diff --git a/jraft-core/pom.xml b/jraft-core/pom.xml new file mode 100644 index 0000000..04ee660 --- /dev/null +++ b/jraft-core/pom.xml @@ -0,0 +1,131 @@ + + + + 4.0.0 + + jraft-parent + com.alipay.sofa + 1.3.10.bugfix + + jraft-core + jar + jraft-core ${project.version} + + + + + junit + junit + test + + + junit + junit-dep + test + + + org.ow2.asm + asm + + + com.google.code.findbugs + jsr305 + + + + org.rocksdb + rocksdbjni + + + + net.java.dev.jna + jna + + + + org.jctools + jctools-core + + + + org.mockito + mockito-all + test + + + org.powermock + powermock-api-mockito + test + + + org.powermock + powermock-module-junit4 + test + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-api + test + + + org.apache.logging.log4j + log4j-core + test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + org.apache.logging.log4j + log4j-jcl + test + + + com.lmax + disruptor + + + com.google.protobuf + protobuf-java + + + + commons-io + commons-io + + + commons-lang + commons-lang + + + + com.alipay.sofa + bolt + + + com.alipay.sofa + hessian + + + + io.dropwizard.metrics + metrics-core + + + + org.openjdk.jmh + jmh-core + + + org.openjdk.jmh + jmh-generator-annprocess + + + diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/CliService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/CliService.java new file mode 100644 index 0000000..e364b30 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/CliService.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.CliOptions; + +/** + * Client command-line service + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 4:05:35 PM + */ +public interface CliService extends Lifecycle { + + /** + * Add a new peer into the replicating group which consists of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param peer peer to add + * @return operation status + */ + Status addPeer(final String groupId, final Configuration conf, final PeerId peer); + + /** + * Remove a peer from the replicating group which consists of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param peer peer to remove + * @return operation status + */ + Status removePeer(final String groupId, final Configuration conf, final PeerId peer); + + /** + * Gracefully change the peers of the replication group. + * + * @param groupId the raft group id + * @param conf current configuration + * @param newPeers new peers to change + * @return operation status + */ + Status changePeers(final String groupId, final Configuration conf, final Configuration newPeers); + + /** + * Reset the peer set of the target peer. + * + * @param groupId the raft group id + * @param peer target peer + * @param newPeers new peers to reset + * @return operation status + */ + Status resetPeer(final String groupId, final PeerId peer, final Configuration newPeers); + + /** + * Add some new learners into the replicating group which consists of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param learners learner peers to add + * @return operation status + * @since 1.3.0 + * + */ + Status addLearners(final String groupId, final Configuration conf, final List learners); + + /** + * Remove some learners from the replicating group which consists of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param learners learner peers to remove + * @return operation status + * @since 1.3.0 + * + */ + Status removeLearners(final String groupId, final Configuration conf, final List learners); + + /** + * Converts the specified learner to follower of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param learner learner peer + * @return operation status + * @since 1.3.8 + * + */ + Status learner2Follower(final String groupId, final Configuration conf, final PeerId learner); + + /** + * Update learners set in the replicating group which consists of |conf|. + * return OK status when success. + * + * @param groupId the raft group id + * @param conf current configuration + * @param learners learner peers to set + * @return operation status + * @since 1.3.0 + * + */ + Status resetLearners(final String groupId, final Configuration conf, final List learners); + + /** + * Transfer the leader of the replication group to the target peer + * + * @param groupId the raft group id + * @param conf current configuration + * @param peer target peer of new leader + * @return operation status + */ + Status transferLeader(final String groupId, final Configuration conf, final PeerId peer); + + /** + * Ask the peer to dump a snapshot immediately. + * + * @param groupId the raft group id + * @param peer target peer + * @return operation status + */ + Status snapshot(final String groupId, final PeerId peer); + + /** + * Get the leader of the replication group. + * @param groupId the raft group id + * @param conf configuration + * @param leaderId id of leader + * @return operation status + */ + Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId); + + /** + * Ask all peers of the replication group. + * + * @param groupId the raft group id + * @param conf target peers configuration + * @return all peers of the replication group + */ + List getPeers(final String groupId, final Configuration conf); + + /** + * Ask all alive peers of the replication group. + * + * @param groupId the raft group id + * @param conf target peers configuration + * @return all alive peers of the replication group + */ + List getAlivePeers(final String groupId, final Configuration conf); + + /** + * Ask all learners of the replication group. + * + * @param groupId the raft group id + * @param conf target peers configuration + * @return all learners of the replication group + * @since 1.3.0 + */ + List getLearners(final String groupId, final Configuration conf); + + /** + * Ask all alive learners of the replication group. + * + * @param groupId the raft group id + * @param conf target peers configuration + * @return all alive learners of the replication group + */ + List getAliveLearners(final String groupId, final Configuration conf); + + /** + * Balance the number of leaders. + * + * @param balanceGroupIds all raft group ids to balance + * @param conf configuration of all nodes + * @param balancedLeaderIds the result of all balanced leader ids + * @return operation status + */ + Status rebalance(final Set balanceGroupIds, final Configuration conf, + final Map balancedLeaderIds); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/Closure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/Closure.java new file mode 100644 index 0000000..7d51264 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/Closure.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +/** + * Callback closure. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:07:05 AM + */ +public interface Closure { + + /** + * Called when task is done. + * + * @param status the task status. + */ + void run(final Status status); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/FSMCaller.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/FSMCaller.java new file mode 100644 index 0000000..71b003e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/FSMCaller.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.closure.LoadSnapshotClosure; +import com.alipay.sofa.jraft.closure.SaveSnapshotClosure; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.FSMCallerOptions; +import com.alipay.sofa.jraft.util.Describer; + +/** + * Finite state machine caller. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:07:52 AM + */ +public interface FSMCaller extends Lifecycle, Describer { + + /** + * Listen on lastAppliedLogIndex update events. + * + * @author dennis + */ + interface LastAppliedLogIndexListener { + + /** + * Called when lastAppliedLogIndex updated. + * + * @param lastAppliedLogIndex the log index of last applied + */ + void onApplied(final long lastAppliedLogIndex); + } + + /** + * Adds a LastAppliedLogIndexListener. + */ + void addLastAppliedLogIndexListener(final LastAppliedLogIndexListener listener); + + /** + * Called when log entry committed + * + * @param committedIndex committed log index + */ + boolean onCommitted(final long committedIndex); + + /** + * Called after loading snapshot. + * + * @param done callback + */ + boolean onSnapshotLoad(final LoadSnapshotClosure done); + + /** + * Called after saving snapshot. + * + * @param done callback + */ + boolean onSnapshotSave(final SaveSnapshotClosure done); + + /** + * Called when the leader stops. + * + * @param status status info + */ + boolean onLeaderStop(final Status status); + + /** + * Called when the leader starts. + * + * @param term current term + */ + boolean onLeaderStart(final long term); + + /** + * Called when start following a leader. + * + * @param ctx context of leader change + */ + boolean onStartFollowing(final LeaderChangeContext ctx); + + /** + * Called when stop following a leader. + * + * @param ctx context of leader change + */ + boolean onStopFollowing(final LeaderChangeContext ctx); + + /** + * Called when error happens. + * + * @param error error info + */ + boolean onError(final RaftException error); + + /** + * Returns the last log entry index to apply state machine. + */ + long getLastAppliedIndex(); + + /** + * Called after shutdown to wait it terminates. + * + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + void join() throws InterruptedException; +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/Iterator.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/Iterator.java new file mode 100644 index 0000000..c80de1f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/Iterator.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.nio.ByteBuffer; + +/** + * Iterator over a batch of committed tasks. + * @see StateMachine#onApply(Iterator) + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 3:20:15 PM + */ +public interface Iterator extends java.util.Iterator { + + /** + * Return the data whose content is the same as what was passed to + * Node#apply(Task) in the leader node. + */ + ByteBuffer getData(); + + /** + * Return a unique and monotonically increasing identifier of the current task: + * - Uniqueness guarantees that committed tasks in different peers with + * the same index are always the same and kept unchanged. + * - Monotonicity guarantees that for any index pair i, j (i < j), task + * at index |i| must be applied before task at index |j| in all the + * peers from the group. + */ + long getIndex(); + + /** + * Returns the term of the leader which to task was applied to. + */ + long getTerm(); + + /** + * If done() is non-NULL, you must call done()->Run() after applying this + * task no matter this operation succeeds or fails, otherwise the + * corresponding resources would leak. + * + * If this task is proposed by this Node when it was the leader of this + * group and the leadership has not changed before this point, done() is + * exactly what was passed to Node#apply(Task) which may stand for some + * continuation (such as respond to the client) after updating the + * StateMachine with the given task. Otherwise done() must be NULL. + * */ + Closure done(); + + /** + * Invoked when some critical error occurred. And we will consider the last + * |ntail| tasks (starting from the last iterated one) as not applied. After + * this point, no further changes on the StateMachine as well as the Node + * would be allowed and you should try to repair this replica or just drop it. + * + * @param ntail the number of tasks (starting from the last iterated one) considered as not to be applied. + * @param st Status to describe the detail of the error. + */ + void setErrorAndRollback(final long ntail, final Status st); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftServiceFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftServiceFactory.java new file mode 100644 index 0000000..84a78fc --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftServiceFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.LogStorage; +import com.alipay.sofa.jraft.storage.RaftMetaStorage; +import com.alipay.sofa.jraft.storage.SnapshotStorage; + +/** + * Abstract factory to create services for SOFAJRaft. + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + */ +public interface JRaftServiceFactory { + + /** + * Creates a raft log storage. + * @param uri The log storage uri from {@link NodeOptions#getSnapshotUri()} + * @param raftOptions the raft options. + * @return storage to store raft log entries. + */ + LogStorage createLogStorage(final String uri, final RaftOptions raftOptions); + + /** + * Creates a raft snapshot storage + * @param uri The snapshot storage uri from {@link NodeOptions#getSnapshotUri()} + * @param raftOptions the raft options. + * @return storage to store state machine snapshot. + */ + SnapshotStorage createSnapshotStorage(final String uri, final RaftOptions raftOptions); + + /** + * Creates a raft meta storage. + * @param uri The meta storage uri from {@link NodeOptions#getRaftMetaUri()} + * @param raftOptions the raft options. + * @return meta storage to store raft meta info. + */ + RaftMetaStorage createRaftMetaStorage(final String uri, final RaftOptions raftOptions); + + /** + * Creates a log entry codec factory. + * @return a codec factory to create encoder/decoder for raft log entry. + */ + LogEntryCodecFactory createLogEntryCodecFactory(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftUtils.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftUtils.java new file mode 100644 index 0000000..020137f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/JRaftUtils.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.concurrent.Executor; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.lang.StringUtils; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.BootstrapOptions; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.JRaftServiceLoader; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.alipay.sofa.jraft.util.timer.RaftTimerFactory; + +/** + * Some helper methods for jraft usage. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-23 3:48:45 PM + */ +public final class JRaftUtils { + + private final static RaftTimerFactory TIMER_FACTORY = JRaftServiceLoader.load(RaftTimerFactory.class) // + .first(); + + /** + * Get raft timer factory. + * + * @return {@link RaftTimerFactory} + */ + public static RaftTimerFactory raftTimerFactory() { + return TIMER_FACTORY; + } + + /** + * Bootstrap a non-empty raft node. + * + * @param opts options of bootstrap + * @return true if bootstrap success + */ + public static boolean bootstrap(final BootstrapOptions opts) throws InterruptedException { + final NodeImpl node = new NodeImpl(); + final boolean ret = node.bootstrap(opts); + node.shutdown(); + node.join(); + return ret; + } + + /** + * Create a executor with size. + * + * @param prefix thread name prefix + * @param number thread number + * @return a new {@link ThreadPoolExecutor} instance + */ + public static Executor createExecutor(final String prefix, final int number) { + if (number <= 0) { + return null; + } + return ThreadPoolUtil.newBuilder() // + .poolName(prefix) // + .enableMetric(true) // + .coreThreads(number) // + .maximumThreads(number) // + .keepAliveSeconds(60L) // + .workQueue(new SynchronousQueue<>()) // + .threadFactory(createThreadFactory(prefix)) // + .build(); + } + + /** + * Create a thread factory. + * + * @param prefixName the prefix name of thread + * @return a new {@link ThreadFactory} instance + * + * @since 0.0.3 + */ + public static ThreadFactory createThreadFactory(final String prefixName) { + return new NamedThreadFactory(prefixName, true); + } + + /** + * Create a configuration from a string in the form of "host1:port1[:idx],host2:port2[:idx]......", + * returns a empty configuration when string is blank. + */ + public static Configuration getConfiguration(final String s) { + final Configuration conf = new Configuration(); + if (StringUtils.isBlank(s)) { + return conf; + } + if (conf.parse(s)) { + return conf; + } + throw new IllegalArgumentException("Invalid conf str:" + s); + } + + /** + * Create a peer from a string in the form of "host:port[:idx]", + * returns a empty peer when string is blank. + */ + public static PeerId getPeerId(final String s) { + final PeerId peer = new PeerId(); + if (StringUtils.isBlank(s)) { + return peer; + } + if (peer.parse(s)) { + return peer; + } + throw new IllegalArgumentException("Invalid peer str:" + s); + } + + /** + * Create a Endpoint instance from a string in the form of "host:port", + * returns null when string is blank. + */ + public static Endpoint getEndPoint(final String s) { + if (StringUtils.isBlank(s)) { + return null; + } + final String[] tmps = StringUtils.split(s, ':'); + if (tmps.length != 2) { + throw new IllegalArgumentException("Invalid endpoint string: " + s); + } + return new Endpoint(tmps[0], Integer.parseInt(tmps[1])); + } + + private JRaftUtils() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/Lifecycle.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/Lifecycle.java new file mode 100644 index 0000000..85bff23 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/Lifecycle.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +/** + * Service life cycle mark interface. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:47:04 PM + */ +public interface Lifecycle { + + /** + * Initialize the service. + * + * @return true when successes. + */ + boolean init(final T opts); + + /** + * Dispose the resources for service. + */ + void shutdown(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/Node.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/Node.java new file mode 100644 index 0000000..990d131 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/Node.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.List; + +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.alipay.sofa.jraft.core.Replicator; +import com.alipay.sofa.jraft.core.State; +import com.alipay.sofa.jraft.entity.NodeId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.entity.UserLog; +import com.alipay.sofa.jraft.error.LogIndexOutOfBoundsException; +import com.alipay.sofa.jraft.error.LogNotFoundException; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.util.Describer; + +/** + * A raft replica node. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:06:55 PM + */ +public interface Node extends Lifecycle, Describer { + + /** + * Get the leader peer id for redirect, null if absent. + */ + PeerId getLeaderId(); + + /** + * Get current node id. + */ + NodeId getNodeId(); + + /** + * Get the node metrics, only valid when node option {@link NodeOptions#isEnableMetrics()} is true. + */ + NodeMetrics getNodeMetrics(); + + /** + * Get the raft group id. + */ + String getGroupId(); + + /** + * Get the node options. + */ + NodeOptions getOptions(); + + /** + * Get the raft options + */ + RaftOptions getRaftOptions(); + + /** + * Returns true when the node is leader. + */ + boolean isLeader(); + + /** + * Returns true when the node is leader. + * @param blocking if true, will be blocked until the node finish it's state change + */ + boolean isLeader(final boolean blocking); + + /** + * Shutdown local replica node. + * + * @param done callback + */ + void shutdown(final Closure done); + + /** + * Block the thread until the node is successfully stopped. + * + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + void join() throws InterruptedException; + + /** + * [Thread-safe and wait-free] + * + * Apply task to the replicated-state-machine + * + * About the ownership: + * |task.data|: for the performance consideration, we will take away the + * content. If you want keep the content, copy it before call + * this function + * |task.done|: If the data is successfully committed to the raft group. We + * will pass the ownership to #{@link StateMachine#onApply(Iterator)}. + * Otherwise we will specify the error and call it. + * + * @param task task to apply + */ + void apply(final Task task); + + /** + * [Thread-safe and wait-free] + * + * Starts a linearizable read-only query request with request context(optional, + * such as request id etc.) and closure. The closure will be called when the + * request is completed, and user can read data from state machine if the result + * status is OK. + * + * @param requestContext the context of request + * @param done callback + * + * @since 0.0.3 + */ + void readIndex(final byte[] requestContext, final ReadIndexClosure done); + + /** + * List peers of this raft group, only leader returns. + * + * [NOTE] when list_peers concurrency with {@link #addPeer(PeerId, Closure)}/{@link #removePeer(PeerId, Closure)}, + * maybe return peers is staled. Because {@link #addPeer(PeerId, Closure)}/{@link #removePeer(PeerId, Closure)} + * immediately modify configuration in memory + * + * @return the peer list + */ + List listPeers(); + + /** + * List all alive peers of this raft group, only leader returns.

+ * + * [NOTE] list_alive_peers is just a transient data (snapshot) + * and a short-term loss of response by the follower will cause it to + * temporarily not exist in this list. + * + * @return the alive peer list + * @since 1.2.6 + */ + List listAlivePeers(); + + /** + * List all learners of this raft group, only leader returns.

+ * + * [NOTE] when listLearners concurrency with {@link #addLearners(List, Closure)}/{@link #removeLearners(List, Closure)}/{@link #resetLearners(List, Closure)}, + * maybe return peers is staled. Because {@link #addLearners(List, Closure)}/{@link #removeLearners(List, Closure)}/{@link #resetLearners(List, Closure)} + * immediately modify configuration in memory + * + * @return the learners set + * @since 1.3.0 + */ + List listLearners(); + + /** + * List all alive learners of this raft group, only leader returns.

+ * + * [NOTE] when listAliveLearners concurrency with {@link #addLearners(List, Closure)}/{@link #removeLearners(List, Closure)}/{@link #resetLearners(List, Closure)}, + * maybe return peers is staled. Because {@link #addLearners(List, Closure)}/{@link #removeLearners(List, Closure)}/{@link #resetLearners(List, Closure)} + * immediately modify configuration in memory + * + * @return the alive learners set + * @since 1.3.0 + */ + List listAliveLearners(); + + /** + * Add a new peer to the raft group. done.run() would be invoked after this + * operation finishes, describing the detailed result. + * + * @param peer peer to add + * @param done callback + */ + void addPeer(final PeerId peer, final Closure done); + + /** + * Remove the peer from the raft group. done.run() would be invoked after + * operation finishes, describing the detailed result. + * + * @param peer peer to remove + * @param done callback + */ + void removePeer(final PeerId peer, final Closure done); + + /** + * Change the configuration of the raft group to |newPeers| , done.run() + * would be invoked after this operation finishes, describing the detailed result. + * + * @param newPeers new peers to change + * @param done callback + */ + void changePeers(final Configuration newPeers, final Closure done); + + /** + * Reset the configuration of this node individually, without any replication + * to other peers before this node becomes the leader. This function is + * supposed to be invoked when the majority of the replication group are + * dead and you'd like to revive the service in the consideration of + * availability. + * Notice that neither consistency nor consensus are guaranteed in this + * case, BE CAREFUL when dealing with this method. + * + * @param newPeers new peers + */ + Status resetPeers(final Configuration newPeers); + + /** + * Add some new learners to the raft group. done.run() will be invoked after this + * operation finishes, describing the detailed result. + * + * @param learners learners to add + * @param done callback + * @since 1.3.0 + */ + void addLearners(final List learners, final Closure done); + + /** + * Remove some learners from the raft group. done.run() will be invoked after this + * operation finishes, describing the detailed result. + * + * @param learners learners to remove + * @param done callback + * @since 1.3.0 + */ + void removeLearners(final List learners, final Closure done); + + /** + * Reset learners in the raft group. done.run() will be invoked after this + * operation finishes, describing the detailed result. + * + * @param learners learners to set + * @param done callback + * @since 1.3.0 + */ + void resetLearners(final List learners, final Closure done); + + /** + * Start a snapshot immediately if possible. done.run() would be invoked when + * the snapshot finishes, describing the detailed result. + * + * @param done callback + */ + void snapshot(final Closure done); + + /** + * Reset the election_timeout for the every node. + * + * @param electionTimeoutMs the timeout millis of election + */ + void resetElectionTimeoutMs(final int electionTimeoutMs); + + /** + * Try transferring leadership to |peer|. If peer is ANY_PEER, a proper follower + * will be chosen as the leader for the next term. + * Returns 0 on success, -1 otherwise. + * + * @param peer the target peer of new leader + * @return operation status + */ + Status transferLeadershipTo(final PeerId peer); + + /** + * Read the first committed user log from the given index. + * Return OK on success and user_log is assigned with the very data. Be awared + * that the user_log may be not the exact log at the given index, but the + * first available user log from the given index to lastCommittedIndex. + * Otherwise, appropriate errors are returned: + * - return ELOGDELETED when the log has been deleted; + * - return ENOMOREUSERLOG when we can't get a user log even reaching lastCommittedIndex. + * [NOTE] in consideration of safety, we use lastAppliedIndex instead of lastCommittedIndex + * in code implementation. + * + * @param index log index + * @return user log entry + * @throws LogNotFoundException the user log is deleted at index. + * @throws LogIndexOutOfBoundsException the special index is out of bounds. + */ + UserLog readCommittedUserLog(final long index); + + /** + * SOFAJRaft users can implement the ReplicatorStateListener interface by themselves. + * So users can do their own logical operator in this listener when replicator created, destroyed or had some errors. + * + * @param replicatorStateListener added ReplicatorStateListener which is implemented by users. + */ + void addReplicatorStateListener(final Replicator.ReplicatorStateListener replicatorStateListener); + + /** + * End User can remove their implement the ReplicatorStateListener interface by themselves. + * + * @param replicatorStateListener need to remove the ReplicatorStateListener which has been added by users. + */ + void removeReplicatorStateListener(final Replicator.ReplicatorStateListener replicatorStateListener); + + /** + * Remove all the ReplicatorStateListeners which have been added by users. + * + */ + void clearReplicatorStateListeners(); + + /** + * Get the ReplicatorStateListeners which have been added by users. + * + * @return node's replicatorStatueListeners which have been added by users. + */ + List getReplicatorStatueListeners(); + + /** + * Get the node's target election priority value. + * + * @return node's target election priority value. + * @since 1.3.0 + */ + int getNodeTargetPriority(); + + /** + * Get the node's state. + * + * @return node's state. + * @since 1.3.8 + */ + State getNodeState(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeDescribeSignalHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeDescribeSignalHandler.java new file mode 100644 index 0000000..42f6910 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeDescribeSignalHandler.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.FileOutputSignalHandler; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; + +/** + * + * @author jiachun.fjc + */ +public class NodeDescribeSignalHandler extends FileOutputSignalHandler { + + private static Logger LOG = LoggerFactory.getLogger(NodeDescribeSignalHandler.class); + + private static final String DIR = SystemPropertyUtil.get("jraft.signal.node.describe.dir", ""); + private static final String BASE_NAME = "node_describe.log"; + + @Override + public void handle(final String signalName) { + final List nodes = NodeManager.getInstance().getAllNodes(); + if (nodes.isEmpty()) { + return; + } + + try { + final File file = getOutputFile(DIR, BASE_NAME); + + LOG.info("Describing raft nodes with signal: {} to file: {}.", signalName, file); + + try (final PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file, true), + StandardCharsets.UTF_8))) { + final Describer.Printer printer = new Describer.DefaultPrinter(out); + for (final Node node : nodes) { + node.describe(printer); + } + } + } catch (final IOException e) { + LOG.error("Fail to describe nodes: {}.", nodes, e); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeManager.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeManager.java new file mode 100644 index 0000000..236ebf7 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeManager.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.stream.Collectors; + +import com.alipay.sofa.jraft.entity.NodeId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Utils; +import com.alipay.sofa.jraft.util.concurrent.ConcurrentHashSet; + +/** + * Raft nodes manager. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-22 5:58:23 PM + */ +public class NodeManager { + + private static final NodeManager INSTANCE = new NodeManager(); + + private final ConcurrentMap nodeMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> groupMap = new ConcurrentHashMap<>(); + private final ConcurrentHashSet addrSet = new ConcurrentHashSet<>(); + + public static NodeManager getInstance() { + return INSTANCE; + } + + /** + * Return true when RPC service is registered. + */ + public boolean serverExists(final Endpoint addr) { + if (addr.getIp().equals(Utils.IP_ANY)) { + return this.addrSet.contains(new Endpoint(Utils.IP_ANY, addr.getPort())); + } + return this.addrSet.contains(addr); + } + + /** + * Remove a RPC service address. + */ + public boolean removeAddress(final Endpoint addr) { + return this.addrSet.remove(addr); + } + + /** + * Adds a RPC service address. + */ + public void addAddress(final Endpoint addr) { + this.addrSet.add(addr); + } + + /** + * Adds a node. + */ + public boolean add(final Node node) { + // check address ok? + if (!serverExists(node.getNodeId().getPeerId().getEndpoint())) { + return false; + } + final NodeId nodeId = node.getNodeId(); + if (this.nodeMap.putIfAbsent(nodeId, node) == null) { + final String groupId = node.getGroupId(); + List nodes = this.groupMap.get(groupId); + if (nodes == null) { + nodes = Collections.synchronizedList(new ArrayList<>()); + List existsNode = this.groupMap.putIfAbsent(groupId, nodes); + if (existsNode != null) { + nodes = existsNode; + } + } + nodes.add(node); + return true; + } + return false; + } + + /** + * Clear the states, for test + */ + @OnlyForTest + public void clear() { + this.groupMap.clear(); + this.nodeMap.clear(); + this.addrSet.clear(); + } + + /** + * Remove a node. + */ + public boolean remove(final Node node) { + if (this.nodeMap.remove(node.getNodeId(), node)) { + final List nodes = this.groupMap.get(node.getGroupId()); + if (nodes != null) { + return nodes.remove(node); + } + } + return false; + } + + /** + * Get node by groupId and peer. + */ + public Node get(final String groupId, final PeerId peerId) { + return this.nodeMap.get(new NodeId(groupId, peerId)); + } + + /** + * Get all nodes in a raft group. + */ + public List getNodesByGroupId(final String groupId) { + return this.groupMap.get(groupId); + } + + /** + * Get all nodes + */ + public List getAllNodes() { + return this.groupMap.values().stream().flatMap(Collection::stream).collect(Collectors.toList()); + } + + private NodeManager() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeMetricsSignalHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeMetricsSignalHandler.java new file mode 100644 index 0000000..2d128cb --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/NodeMetricsSignalHandler.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.alipay.sofa.jraft.util.FileOutputSignalHandler; +import com.alipay.sofa.jraft.util.MetricReporter; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.codahale.metrics.MetricRegistry; + +/** + * + * @author jiachun.fjc + */ +public class NodeMetricsSignalHandler extends FileOutputSignalHandler { + + private static Logger LOG = LoggerFactory.getLogger(NodeMetricsSignalHandler.class); + + private static final String DIR = SystemPropertyUtil.get("jraft.signal.node.metrics.dir", ""); + private static final String BASE_NAME = "node_metrics.log"; + + @Override + public void handle(final String signalName) { + final List nodes = NodeManager.getInstance().getAllNodes(); + if (nodes.isEmpty()) { + return; + } + + try { + final File file = getOutputFile(DIR, BASE_NAME); + + LOG.info("Printing raft nodes metrics with signal: {} to file: {}.", signalName, file); + + try (final PrintStream out = new PrintStream(new FileOutputStream(file, true))) { + for (final Node node : nodes) { + final NodeMetrics nodeMetrics = node.getNodeMetrics(); + final MetricRegistry registry = nodeMetrics.getMetricRegistry(); + if (registry == null) { + LOG.warn("Node: {} received a signal to print metric, but it does not have metric enabled.", + node); + continue; + } + final MetricReporter reporter = MetricReporter.forRegistry(registry) // + .outputTo(out) // + .prefixedWith("-- " + node.getNodeId()) // + .build(); + reporter.report(); + } + } + } catch (final IOException e) { + LOG.error("Fail to print nodes metrics: {}.", nodes, e); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftGroupService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftGroupService.java new file mode 100644 index 0000000..d206c2c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftGroupService.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.ProtobufMsgFactory; +import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory; +import com.alipay.sofa.jraft.rpc.RpcServer; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.Utils; + +/** + * A framework to implement a raft group service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 7:53:03 PM + */ +public class RaftGroupService { + + private static final Logger LOG = LoggerFactory.getLogger(RaftGroupService.class); + + static { + ProtobufMsgFactory.load(); + } + + private volatile boolean started = false; + + /** + * This node serverId + */ + private PeerId serverId; + + /** + * Node options + */ + private NodeOptions nodeOptions; + + /** + * The raft RPC server + */ + private RpcServer rpcServer; + + /** + * If we want to share the rpcServer instance, then we can't stop it when shutdown. + */ + private final boolean sharedRpcServer; + + /** + * The raft group id + */ + private String groupId; + /** + * The raft node. + */ + private Node node; + + public RaftGroupService(final String groupId, final PeerId serverId, final NodeOptions nodeOptions) { + this(groupId, serverId, nodeOptions, RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint(), + JRaftUtils.createExecutor("RAFT-RPC-executor-", nodeOptions.getRaftRpcThreadPoolSize()), + JRaftUtils.createExecutor("CLI-RPC-executor-", nodeOptions.getCliRpcThreadPoolSize()))); + } + + public RaftGroupService(final String groupId, final PeerId serverId, final NodeOptions nodeOptions, + final RpcServer rpcServer) { + this(groupId, serverId, nodeOptions, rpcServer, false); + } + + public RaftGroupService(final String groupId, final PeerId serverId, final NodeOptions nodeOptions, + final RpcServer rpcServer, final boolean sharedRpcServer) { + super(); + this.groupId = groupId; + this.serverId = serverId; + this.nodeOptions = nodeOptions; + this.rpcServer = rpcServer; + this.sharedRpcServer = sharedRpcServer; + } + + public synchronized Node getRaftNode() { + return this.node; + } + + /** + * Starts the raft group service, returns the raft node. + */ + public synchronized Node start() { + return start(true); + } + + /** + * Starts the raft group service, returns the raft node. + * + * @param startRpcServer whether to start RPC server. + */ + public synchronized Node start(final boolean startRpcServer) { + if (this.started) { + return this.node; + } + if (this.serverId == null || this.serverId.getEndpoint() == null + || this.serverId.getEndpoint().equals(new Endpoint(Utils.IP_ANY, 0))) { + throw new IllegalArgumentException("Blank serverId:" + this.serverId); + } + if (StringUtils.isBlank(this.groupId)) { + throw new IllegalArgumentException("Blank group id:" + this.groupId); + } + //Adds RPC server to Server. + NodeManager.getInstance().addAddress(this.serverId.getEndpoint()); + + this.node = RaftServiceFactory.createAndInitRaftNode(this.groupId, this.serverId, this.nodeOptions); + if (startRpcServer) { + this.rpcServer.init(null); + } else { + LOG.warn("RPC server is not started in RaftGroupService."); + } + this.started = true; + LOG.info("Start the RaftGroupService successfully."); + return this.node; + } + + /** + * Block thread to wait the server shutdown. + * + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + public synchronized void join() throws InterruptedException { + if (this.node != null) { + this.node.join(); + this.node = null; + } + } + + public synchronized void shutdown() { + if (!this.started) { + return; + } + if (this.rpcServer != null) { + try { + if (!this.sharedRpcServer) { + this.rpcServer.shutdown(); + } + } catch (final Exception ignored) { + // ignore + } + this.rpcServer = null; + } + this.node.shutdown(); + NodeManager.getInstance().removeAddress(this.serverId.getEndpoint()); + this.started = false; + LOG.info("Stop the RaftGroupService successfully."); + } + + /** + * Returns true when service is started. + */ + public boolean isStarted() { + return this.started; + } + + /** + * Returns the raft group id. + */ + public String getGroupId() { + return this.groupId; + } + + /** + * Set the raft group id + */ + public void setGroupId(final String groupId) { + if (this.started) { + throw new IllegalStateException("Raft group service already started"); + } + this.groupId = groupId; + } + + /** + * Returns the node serverId + */ + public PeerId getServerId() { + return this.serverId; + } + + /** + * Set the node serverId + */ + public void setServerId(final PeerId serverId) { + if (this.started) { + throw new IllegalStateException("Raft group service already started"); + } + this.serverId = serverId; + } + + /** + * Returns the node options. + */ + public RpcOptions getNodeOptions() { + return this.nodeOptions; + } + + /** + * Set node options. + */ + public void setNodeOptions(final NodeOptions nodeOptions) { + if (this.started) { + throw new IllegalStateException("Raft group service already started"); + } + if (nodeOptions == null) { + throw new IllegalArgumentException("Invalid node options."); + } + nodeOptions.validate(); + this.nodeOptions = nodeOptions; + } + + /** + * Returns the rpc server instance. + */ + public RpcServer getRpcServer() { + return this.rpcServer; + } + + /** + * Set rpc server. + */ + public void setRpcServer(final RpcServer rpcServer) { + if (this.started) { + throw new IllegalStateException("Raft group service already started"); + } + if (this.serverId == null) { + throw new IllegalStateException("Please set serverId at first"); + } + if (rpcServer.boundPort() != this.serverId.getPort()) { + throw new IllegalArgumentException("RPC server port mismatch"); + } + this.rpcServer = rpcServer; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftServiceFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftServiceFactory.java new file mode 100644 index 0000000..8aaea5c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/RaftServiceFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.core.CliServiceImpl; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.option.NodeOptions; + +/** + * Service factory to create raft services, such as Node/CliService etc. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-May-03 11:06:02 AM + */ +public final class RaftServiceFactory { + + /** + * Create a raft node with group id and it's serverId. + */ + public static Node createRaftNode(final String groupId, final PeerId serverId) { + return new NodeImpl(groupId, serverId); + } + + /** + * Create and initialize a raft node with node options. + * Throw {@link IllegalStateException} when fail to initialize. + */ + public static Node createAndInitRaftNode(final String groupId, final PeerId serverId, final NodeOptions opts) { + final Node ret = createRaftNode(groupId, serverId); + if (!ret.init(opts)) { + throw new IllegalStateException("Fail to init node, please see the logs to find the reason."); + } + return ret; + } + + /** + * Create a {@link CliService} instance. + */ + public static CliService createCliService() { + return new CliServiceImpl(); + } + + /** + * Create and initialize a CliService instance. + */ + public static CliService createAndInitCliService(final CliOptions cliOptions) { + final CliService ret = createCliService(); + if (!ret.init(cliOptions)) { + throw new IllegalStateException("Fail to init CliService"); + } + return ret; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/ReadOnlyService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/ReadOnlyService.java new file mode 100644 index 0000000..cd14364 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/ReadOnlyService.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.ReadOnlyServiceOptions; + +/** + * The read-only query service. + * + * @author dennis + * + */ +public interface ReadOnlyService extends Lifecycle { + + /** + * Adds a ReadIndex request. + * + * @param reqCtx request context of readIndex + * @param closure callback + */ + void addRequest(final byte[] reqCtx, final ReadIndexClosure closure); + + /** + * Waits for service shutdown. + * + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + void join() throws InterruptedException; + + /** + * Called when the node is turned into error state. + * @param error error with raft info + */ + void setError(final RaftException error); + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/ReplicatorGroup.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/ReplicatorGroup.java new file mode 100644 index 0000000..7054194 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/ReplicatorGroup.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.List; + +import com.alipay.sofa.jraft.closure.CatchUpClosure; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.core.ReplicatorType; +import com.alipay.sofa.jraft.entity.NodeId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.ReplicatorGroupOptions; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.ThreadId; + +/** + * Replicators in a raft group. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 5:35:26 PM + */ +public interface ReplicatorGroup extends Describer { + /** + * Init the replicator group. + * + * @param nodeId node id + * @param opts options of replicator group + * @return true if init success + */ + boolean init(final NodeId nodeId, final ReplicatorGroupOptions opts); + + /** + * Adds a replicator for follower({@link ReplicatorType#Follower}). + * @see #addReplicator(PeerId, ReplicatorType) + * + * @param peer target peer + * @return true on success + */ + default boolean addReplicator(final PeerId peer) { + return addReplicator(peer, ReplicatorType.Follower); + } + + /** + * Add a replicator attached with |peer| + * will be a notification when the replicator catches up according to the + * arguments. + * NOTE: when calling this function, the replicators starts to work + * immediately, and might call Node#stepDown which might have race with + * the caller, you should deal with this situation. + * + * @param peer target peer + * @param replicatorType replicator type + * @return true on success + */ + default boolean addReplicator(final PeerId peer, ReplicatorType replicatorType) { + return addReplicator(peer, replicatorType, true); + } + + /** + * Try to add a replicator attached with |peer| + * will be a notification when the replicator catches up according to the + * arguments. + * NOTE: when calling this function, the replicators starts to work + * immediately, and might call Node#stepDown which might have race with + * the caller, you should deal with this situation. + * + * @param peer target peer + * @param replicatorType replicator type + * @param sync synchronous + * @return true on success + */ + boolean addReplicator(final PeerId peer, ReplicatorType replicatorType, boolean sync); + + /** + * Send heartbeat to a peer. + * + * @param peer target peer + * @param closure callback + */ + void sendHeartbeat(final PeerId peer, final RpcResponseClosure closure); + + /** + * Get replicator id by peer, null if not found. + * + * @param peer peer of replicator + * @return the replicator id + */ + ThreadId getReplicator(final PeerId peer); + + /** + * Check replicator state, if it's not started, start it; + * if it is blocked, unblock it. It should be called by leader. + * + * @param peer peer of replicator + * @param lockNode if lock with node + */ + void checkReplicator(final PeerId peer, final boolean lockNode); + + /** + * Clear failure to start replicators + */ + void clearFailureReplicators(); + + /** + * Wait the peer catchup. + */ + boolean waitCaughtUp(final PeerId peer, final long maxMargin, final long dueTime, final CatchUpClosure done); + + /** + * Get peer's last rpc send timestamp (monotonic time in milliseconds). + * + * @param peer the peer of replicator + */ + long getLastRpcSendTimestamp(final PeerId peer); + + /** + * Stop all replicators. + */ + boolean stopAll(); + + /** + * Stop replicator for the peer. + * + * @param peer the peer of replicator + * @return true on success + */ + boolean stopReplicator(final PeerId peer); + + /** + * Reset the term of all to-add replicators. + * This method is supposed to be called when the very candidate becomes the + * leader, so we suppose that there are no running replicators. + * Return true on success, false otherwise + * + * @param newTerm new term num + * @return true on success + */ + boolean resetTerm(final long newTerm); + + /** + * Reset the interval of heartbeat, + * This method is supposed to be called when the very candidate becomes the + * leader, so we suppose that there are no running replicators. + * return true when success, false otherwise. + * + * @param newIntervalMs new heartbeat interval millis + * @return true on success + */ + boolean resetHeartbeatInterval(final int newIntervalMs); + + /** + * Reset the interval of electionTimeout for replicator. + * + * @param newIntervalMs new election timeout millis + * @return true on success + */ + boolean resetElectionTimeoutInterval(final int newIntervalMs); + + /** + * Returns true if the there's a replicator attached to the given |peer| + * + * @param peer target peer + * @return true on contains + */ + boolean contains(final PeerId peer); + + /** + * Transfer leadership to the given |peer| + * + * @param peer target peer + * @param logIndex log index + * @return true on success + */ + boolean transferLeadershipTo(final PeerId peer, final long logIndex); + + /** + * Stop transferring leadership to the given |peer| + * + * @param peer target peer + * @return true on success + */ + boolean stopTransferLeadership(final PeerId peer); + + /** + * Stop all the replicators except for the one that we think can be the + * candidate of the next leader, which has the largest `last_log_id' among + * peers in |current_conf|. + * |candidate| would be returned if we found one and + * the caller is responsible for stopping it, or an invalid value if we + * found none. + * Returns candidate replicator id on success and null otherwise. + * + * @param conf configuration of all replicators + * @return candidate replicator id on success + */ + ThreadId stopAllAndFindTheNextCandidate(final ConfigurationEntry conf); + + /** + * Find the follower with the most log entries in this group, which is + * likely becomes the leader according to the election algorithm of raft. + * Returns the follower peerId on success and null otherwise. + * + * @param conf configuration of all replicators + * @return the follower peerId on success + */ + PeerId findTheNextCandidate(final ConfigurationEntry conf); + + /** + * Returns all replicators. + */ + List listReplicators(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/RouteTable.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/RouteTable.java new file mode 100644 index 0000000..94578a2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/RouteTable.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.StampedLock; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliClientService; +import com.alipay.sofa.jraft.rpc.CliRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.Requires; +import com.google.protobuf.Message; + +/** + * Maintain routes to raft groups. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 10:41:21 AM + */ +public class RouteTable implements Describer { + + private static final Logger LOG = LoggerFactory.getLogger(RouteTable.class); + + private static final RouteTable INSTANCE = new RouteTable(); + + // Map + private final ConcurrentMap groupConfTable = new ConcurrentHashMap<>(); + + public static RouteTable getInstance() { + return INSTANCE; + } + + /** + * Update configuration of group in route table. + * + * @param groupId raft group id + * @param conf configuration to update + * @return true on success + */ + public boolean updateConfiguration(final String groupId, final Configuration conf) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + + final GroupConf gc = getOrCreateGroupConf(groupId); + final StampedLock stampedLock = gc.stampedLock; + final long stamp = stampedLock.writeLock(); + try { + gc.conf = conf; + if (gc.leader != null && !gc.conf.contains(gc.leader)) { + gc.leader = null; + } + } finally { + stampedLock.unlockWrite(stamp); + } + return true; + } + + private GroupConf getOrCreateGroupConf(final String groupId) { + GroupConf gc = this.groupConfTable.get(groupId); + if (gc == null) { + gc = new GroupConf(); + final GroupConf old = this.groupConfTable.putIfAbsent(groupId, gc); + if (old != null) { + gc = old; + } + } + return gc; + } + + /** + * Update configuration of group in route table. + * + * @param groupId raft group id + * @param confStr configuration string + * @return true on success + */ + public boolean updateConfiguration(final String groupId, final String confStr) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireTrue(!StringUtils.isBlank(confStr), "Blank configuration"); + + final Configuration conf = new Configuration(); + if (conf.parse(confStr)) { + return updateConfiguration(groupId, conf); + } else { + LOG.error("Fail to parse confStr: {}", confStr); + return false; + } + } + + /** + * Get the cached leader of the group, return it when found, null otherwise. + * Make sure calls {@link #refreshLeader(CliClientService, String, int)} already + * before invoke this method. + * + * @param groupId raft group id + * @return peer of leader + */ + public PeerId selectLeader(final String groupId) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + + final GroupConf gc = this.groupConfTable.get(groupId); + if (gc == null) { + return null; + } + final StampedLock stampedLock = gc.stampedLock; + long stamp = stampedLock.tryOptimisticRead(); + PeerId leader = gc.leader; + if (!stampedLock.validate(stamp)) { + stamp = stampedLock.readLock(); + try { + leader = gc.leader; + } finally { + stampedLock.unlockRead(stamp); + } + } + return leader; + } + + /** + * Update leader info. + * + * @param groupId raft group id + * @param leader peer of leader + * @return true on success + */ + public boolean updateLeader(final String groupId, final PeerId leader) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + + if (leader != null) { + // If leader presents, it should not be empty. + Requires.requireTrue(!leader.isEmpty(), "Empty leader"); + } + + final GroupConf gc = getOrCreateGroupConf(groupId); + final StampedLock stampedLock = gc.stampedLock; + final long stamp = stampedLock.writeLock(); + try { + gc.leader = leader; + } finally { + stampedLock.unlockWrite(stamp); + } + return true; + } + + /** + * Update leader info. + * + * @param groupId raft group id + * @param leaderStr peer string of leader + * @return true on success + */ + public boolean updateLeader(final String groupId, final String leaderStr) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireTrue(!StringUtils.isBlank(leaderStr), "Blank leader"); + + final PeerId leader = new PeerId(); + if (leader.parse(leaderStr)) { + return updateLeader(groupId, leader); + } else { + LOG.error("Fail to parse leaderStr: {}", leaderStr); + return false; + } + } + + /** + * Get the configuration by groupId, returns null when not found. + * + * @param groupId raft group id + * @return configuration of the group id + */ + public Configuration getConfiguration(final String groupId) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + + final GroupConf gc = this.groupConfTable.get(groupId); + if (gc == null) { + return null; + } + final StampedLock stampedLock = gc.stampedLock; + long stamp = stampedLock.tryOptimisticRead(); + Configuration conf = gc.conf; + if (!stampedLock.validate(stamp)) { + stamp = stampedLock.readLock(); + try { + conf = gc.conf; + } finally { + stampedLock.unlockRead(stamp); + } + } + return conf; + } + + /** + * Blocking the thread until query_leader finishes. + * + * @param groupId raft group id + * @param timeoutMs timeout millis + * @return operation status + */ + public Status refreshLeader(final CliClientService cliClientService, final String groupId, final int timeoutMs) + throws InterruptedException, + TimeoutException { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireTrue(timeoutMs > 0, "Invalid timeout: " + timeoutMs); + + final Configuration conf = getConfiguration(groupId); + if (conf == null) { + return new Status(RaftError.ENOENT, + "Group %s is not registered in RouteTable, forgot to call updateConfiguration?", groupId); + } + final Status st = Status.OK(); + final CliRequests.GetLeaderRequest.Builder rb = CliRequests.GetLeaderRequest.newBuilder(); + rb.setGroupId(groupId); + final CliRequests.GetLeaderRequest request = rb.build(); + TimeoutException timeoutException = null; + for (final PeerId peer : conf) { + if (!cliClientService.connect(peer.getEndpoint())) { + if (st.isOk()) { + st.setError(-1, "Fail to init channel to %s", peer); + } else { + final String savedMsg = st.getErrorMsg(); + st.setError(-1, "%s, Fail to init channel to %s", savedMsg, peer); + } + continue; + } + final Future result = cliClientService.getLeader(peer.getEndpoint(), request, null); + try { + final Message msg = result.get(timeoutMs, TimeUnit.MILLISECONDS); + if (msg instanceof RpcRequests.ErrorResponse) { + if (st.isOk()) { + st.setError(-1, ((RpcRequests.ErrorResponse) msg).getErrorMsg()); + } else { + final String savedMsg = st.getErrorMsg(); + st.setError(-1, "%s, %s", savedMsg, ((RpcRequests.ErrorResponse) msg).getErrorMsg()); + } + } else { + final CliRequests.GetLeaderResponse response = (CliRequests.GetLeaderResponse) msg; + updateLeader(groupId, response.getLeaderId()); + return Status.OK(); + } + } catch (final TimeoutException e) { + timeoutException = e; + } catch (final ExecutionException e) { + if (st.isOk()) { + st.setError(-1, e.getMessage()); + } else { + final String savedMsg = st.getErrorMsg(); + st.setError(-1, "%s, %s", savedMsg, e.getMessage()); + } + } + } + if (timeoutException != null) { + throw timeoutException; + } + + return st; + } + + public Status refreshConfiguration(final CliClientService cliClientService, final String groupId, + final int timeoutMs) throws InterruptedException, TimeoutException { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireTrue(timeoutMs > 0, "Invalid timeout: " + timeoutMs); + + final Configuration conf = getConfiguration(groupId); + if (conf == null) { + return new Status(RaftError.ENOENT, + "Group %s is not registered in RouteTable, forgot to call updateConfiguration?", groupId); + } + final Status st = Status.OK(); + PeerId leaderId = selectLeader(groupId); + if (leaderId == null) { + refreshLeader(cliClientService, groupId, timeoutMs); + leaderId = selectLeader(groupId); + } + if (leaderId == null) { + st.setError(-1, "Fail to get leader of group %s", groupId); + return st; + } + if (!cliClientService.connect(leaderId.getEndpoint())) { + st.setError(-1, "Fail to init channel to %s", leaderId); + return st; + } + final CliRequests.GetPeersRequest.Builder rb = CliRequests.GetPeersRequest.newBuilder(); + rb.setGroupId(groupId); + rb.setLeaderId(leaderId.toString()); + try { + final Message result = cliClientService.getPeers(leaderId.getEndpoint(), rb.build(), null).get(timeoutMs, + TimeUnit.MILLISECONDS); + if (result instanceof CliRequests.GetPeersResponse) { + final CliRequests.GetPeersResponse resp = (CliRequests.GetPeersResponse) result; + final Configuration newConf = new Configuration(); + for (final String peerIdStr : resp.getPeersList()) { + final PeerId newPeer = new PeerId(); + newPeer.parse(peerIdStr); + newConf.addPeer(newPeer); + } + if (!conf.equals(newConf)) { + LOG.info("Configuration of replication group {} changed from {} to {}", groupId, conf, newConf); + } + updateConfiguration(groupId, newConf); + } else { + final RpcRequests.ErrorResponse resp = (RpcRequests.ErrorResponse) result; + st.setError(resp.getErrorCode(), resp.getErrorMsg()); + } + } catch (final Exception e) { + st.setError(-1, e.getMessage()); + } + return st; + } + + /** + * Reset the states. + */ + public void reset() { + this.groupConfTable.clear(); + } + + /** + * Remove the group from route table. + * + * @param groupId raft group id + * @return true on success + */ + public boolean removeGroup(final String groupId) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + + return this.groupConfTable.remove(groupId) != null; + } + + @Override + public String toString() { + return "RouteTable{" + "groupConfTable=" + groupConfTable + '}'; + } + + private RouteTable() { + } + + @Override + public void describe(final Printer out) { + out.println("RouteTable:") // + .print(" ") // + .println(toString()); + } + + private static class GroupConf { + + private final StampedLock stampedLock = new StampedLock(); + + private Configuration conf; + private PeerId leader; + + @Override + public String toString() { + return "GroupConf{" + "conf=" + conf + ", leader=" + leader + '}'; + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/StateMachine.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/StateMachine.java new file mode 100644 index 0000000..ccf1aa2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/StateMachine.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; + +/** + * |StateMachine| is the sink of all the events of a very raft node. + * Implement a specific StateMachine for your own business logic. + * NOTE: All the interfaces are not guaranteed to be thread safe and they are + * called sequentially, saying that every single operation will block all the + * following ones. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 5:43:21 PM + */ +public interface StateMachine { + + /** + * Update the StateMachine with a batch a tasks that can be accessed + * through |iterator|. + * + * Invoked when one or more tasks that were passed to Node#apply(Task) have been + * committed to the raft group (quorum of the group peers have received + * those tasks and stored them on the backing storage). + * + * Once this function returns to the caller, we will regard all the iterated + * tasks through |iter| have been successfully applied. And if you didn't + * apply all the the given tasks, we would regard this as a critical error + * and report a error whose type is ERROR_TYPE_STATE_MACHINE. + * + * @param iter iterator of states + */ + void onApply(final Iterator iter); + + /** + * Invoked once when the raft node was shut down. + * Default do nothing + */ + void onShutdown(); + + /** + * User defined snapshot generate function, this method will block StateMachine#onApply(Iterator). + * user can make snapshot async when fsm can be cow(copy-on-write). + * call done.run(status) when snapshot finished. + * Default: Save nothing and returns error. + * + * @param writer snapshot writer + * @param done callback + */ + void onSnapshotSave(final SnapshotWriter writer, final Closure done); + + /** + * User defined snapshot load function + * get and load snapshot + * Default: Load nothing and returns error. + * + * @param reader snapshot reader + * @return true on success + */ + boolean onSnapshotLoad(final SnapshotReader reader); + + /** + * Invoked when the belonging node becomes the leader of the group at |term| + * Default: Do nothing + * + * @param term new term num + */ + void onLeaderStart(final long term); + + /** + * Invoked when this node steps down from the leader of the replication + * group and |status| describes detailed information + * + * @param status status info + */ + void onLeaderStop(final Status status); + + /** + * This method is called when a critical error was encountered, after this + * point, no any further modification is allowed to applied to this node + * until the error is fixed and this node restarts. + * + * @param e raft error message + */ + void onError(final RaftException e); + + /** + * Invoked when a configuration has been committed to the group. + * + * @param conf committed configuration + */ + void onConfigurationCommitted(final Configuration conf); + + /** + * This method is called when a follower stops following a leader and its leaderId becomes null, + * situations including: + * 1. handle election timeout and start preVote + * 2. receive requests with higher term such as VoteRequest from a candidate + * or appendEntries request from a new leader + * 3. receive timeoutNow request from current leader and start request vote. + * + * the parameter ctx gives the information(leaderId, term and status) about the + * very leader whom the follower followed before. + * User can reset the node's information as it stops following some leader. + * + * @param ctx context of leader change + */ + void onStopFollowing(final LeaderChangeContext ctx); + + /** + * This method is called when a follower or candidate starts following a leader and its leaderId + * (should be NULL before the method is called) is set to the leader's id, + * situations including: + * 1. a candidate receives appendEntries request from a leader + * 2. a follower(without leader) receives appendEntries from a leader + * + * the parameter ctx gives the information(leaderId, term and status) about + * the very leader whom the follower starts to follow. + * User can reset the node's information as it starts to follow some leader. + * + * @param ctx context of leader change + */ + void onStartFollowing(final LeaderChangeContext ctx); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/Status.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/Status.java new file mode 100644 index 0000000..559a093 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/Status.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.util.Copiable; + +//A Status encapsulates the result of an operation. It may indicate success, + +//or it may indicate an error with an associated error message. It's suitable +//for passing status of functions with richer information than just error_code +//in exception-forbidden code. This utility is inspired by leveldb::Status. +// +//Multiple threads can invoke const methods on a Status without +//external synchronization, but if any of the threads may call a +//non-const method, all threads accessing the same Status must use +//external synchronization. +// +//Since failed status needs to allocate memory, you should be careful when +//failed status is frequent. +public class Status implements Copiable { + + /** + * Status internal state. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:17:51 AM + */ + private static class State { + /** error code */ + int code; + /** error msg*/ + String msg; + + State(int code, String msg) { + super(); + this.code = code; + this.msg = msg; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + this.code; + result = prime * result + (this.msg == null ? 0 : this.msg.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + State other = (State) obj; + if (this.code != other.code) { + return false; + } + if (this.msg == null) { + return other.msg == null; + } else { + return this.msg.equals(other.msg); + } + } + } + + private State state; + + public Status() { + this.state = null; + } + + /** + * Creates a OK status instance. + */ + public static Status OK() { + return new Status(); + } + + public Status(Status s) { + if (s.state != null) { + this.state = new State(s.state.code, s.state.msg); + } else { + this.state = null; + } + } + + public Status(RaftError raftError, String fmt, Object... args) { + this.state = new State(raftError.getNumber(), String.format(fmt, args)); + } + + public Status(int code, String fmt, Object... args) { + this.state = new State(code, String.format(fmt, args)); + } + + public Status(int code, String errorMsg) { + this.state = new State(code, errorMsg); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (this.state == null ? 0 : this.state.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Status other = (Status) obj; + if (this.state == null) { + return other.state == null; + } else { + return this.state.equals(other.state); + } + } + + /** + * Reset status to be OK state. + */ + public void reset() { + this.state = null; + } + + /** + * Returns true when status is in OK state. + */ + public boolean isOk() { + return this.state == null || this.state.code == 0; + } + + /** + * Set error code. + */ + public void setCode(int code) { + if (this.state == null) { + this.state = new State(code, null); + } else { + this.state.code = code; + } + } + + /** + * Get error code. + */ + public int getCode() { + return this.state == null ? 0 : this.state.code; + } + + /** + * Get raft error from error code. + */ + public RaftError getRaftError() { + return this.state == null ? RaftError.SUCCESS : RaftError.forNumber(this.state.code); + } + + /** + * Set error msg + */ + public void setErrorMsg(String errMsg) { + if (this.state == null) { + this.state = new State(0, errMsg); + } else { + this.state.msg = errMsg; + } + } + + /** + * Set error code and error msg. + */ + public void setError(int code, String fmt, Object... args) { + this.state = new State(code, String.format(String.valueOf(fmt), args)); + } + + /** + * Set raft error and error msg. + */ + public void setError(RaftError error, String fmt, Object... args) { + this.state = new State(error.getNumber(), String.format(String.valueOf(fmt), args)); + } + + @Override + public String toString() { + if (isOk()) { + return "Status[OK]"; + } else { + return "Status[" + RaftError.describeCode(this.state.code) + "<" + this.state.code + ">: " + this.state.msg + + "]"; + } + } + + @Override + public Status copy() { + return new Status(this.getCode(), this.getErrorMsg()); + } + + /** + * Get the error msg. + */ + public String getErrorMsg() { + return this.state == null ? null : this.state.msg; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/ThreadPoolMetricsSignalHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/ThreadPoolMetricsSignalHandler.java new file mode 100644 index 0000000..6e89667 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/ThreadPoolMetricsSignalHandler.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.util.FileOutputSignalHandler; +import com.alipay.sofa.jraft.util.MetricReporter; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.alipay.sofa.jraft.util.ThreadPoolMetricRegistry; + +/** + * + * @author jiachun.fjc + */ +public class ThreadPoolMetricsSignalHandler extends FileOutputSignalHandler { + + private static Logger LOG = LoggerFactory.getLogger(ThreadPoolMetricsSignalHandler.class); + + private static final String DIR = SystemPropertyUtil.get("jraft.signal.thread.pool.metrics.dir", ""); + private static final String BASE_NAME = "thread_pool_metrics.log"; + + @Override + public void handle(final String signalName) { + try { + final File file = getOutputFile(DIR, BASE_NAME); + + LOG.info("Printing thread pools metrics with signal: {} to file: {}.", signalName, file); + + try (final PrintStream out = new PrintStream(new FileOutputStream(file, true))) { + MetricReporter.forRegistry(ThreadPoolMetricRegistry.metricRegistry()) // + .outputTo(out) // + .build() // + .report(); + } + } catch (final IOException e) { + LOG.error("Fail to print thread pools metrics.", e); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/CatchUpClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/CatchUpClosure.java new file mode 100644 index 0000000..5e8558f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/CatchUpClosure.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.concurrent.ScheduledFuture; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; + +/** + * A catchup closure for peer to catch up. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:15:05 PM + */ +public abstract class CatchUpClosure implements Closure { + + private long maxMargin; + private ScheduledFuture timer; + private boolean hasTimer; + private boolean errorWasSet; + + private final Status status = Status.OK(); + + public Status getStatus() { + return this.status; + } + + public long getMaxMargin() { + return this.maxMargin; + } + + public void setMaxMargin(long maxMargin) { + this.maxMargin = maxMargin; + } + + public ScheduledFuture getTimer() { + return this.timer; + } + + public void setTimer(ScheduledFuture timer) { + this.timer = timer; + this.hasTimer = true; + } + + public boolean hasTimer() { + return this.hasTimer; + } + + public boolean isErrorWasSet() { + return this.errorWasSet; + } + + public void setErrorWasSet(boolean errorWasSet) { + this.errorWasSet = errorWasSet; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueue.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueue.java new file mode 100644 index 0000000..5aeded5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueue.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.List; + +import javax.annotation.concurrent.ThreadSafe; + +import com.alipay.sofa.jraft.Closure; + +/** + * A thread-safe closure queue. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-14 10:29:12 AM + */ +@ThreadSafe +public interface ClosureQueue { + + /** + * Clear all closure in queue. + */ + void clear(); + + /** + * Reset the first index in queue. + * + * @param firstIndex the first index of queue + */ + void resetFirstIndex(final long firstIndex); + + /** + * Append a new closure into queue. + * + * @param closure the closure to append + */ + void appendPendingClosure(final Closure closure); + + /** + * Pop closure from queue until index(inclusion), returns the first + * popped out index, returns -1 when out of range, returns index+1 + * when not found. + * + * @param endIndex the index of queue + * @param closures closure list + * @return returns the first popped out index, returns -1 when out + * of range, returns index+1 + * when not found. + */ + long popClosureUntil(final long endIndex, final List closures); + + /** + * Pop closure from queue until index(inclusion), returns the first + * popped out index, returns -1 when out of range, returns index+1 + * when not found. + * + * @param endIndex the index of queue + * @param closures closure list + * @param taskClosures task closure list + * @return returns the first popped out index, returns -1 when out + * of range, returns index+1 + * when not found. + */ + long popClosureUntil(final long endIndex, final List closures, final List taskClosures); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueueImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueueImpl.java new file mode 100644 index 0000000..10e9ed0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ClosureQueueImpl.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Closure queue implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-28 11:44:01 AM + */ +public class ClosureQueueImpl implements ClosureQueue { + + private static final Logger LOG = LoggerFactory.getLogger(ClosureQueueImpl.class); + + private final Lock lock; + private long firstIndex; + private LinkedList queue; + + @OnlyForTest + public long getFirstIndex() { + return firstIndex; + } + + @OnlyForTest + public LinkedList getQueue() { + return queue; + } + + public ClosureQueueImpl() { + super(); + this.lock = new ReentrantLock(); + this.firstIndex = 0; + this.queue = new LinkedList<>(); + } + + @Override + public void clear() { + List savedQueue; + this.lock.lock(); + try { + this.firstIndex = 0; + savedQueue = this.queue; + this.queue = new LinkedList<>(); + } finally { + this.lock.unlock(); + } + + final Status status = new Status(RaftError.EPERM, "Leader stepped down"); + Utils.runInThread(() -> { + for (final Closure done : savedQueue) { + if (done != null) { + done.run(status); + } + } + }); + } + + @Override + public void resetFirstIndex(final long firstIndex) { + this.lock.lock(); + try { + Requires.requireTrue(this.queue.isEmpty(), "Queue is not empty."); + this.firstIndex = firstIndex; + } finally { + this.lock.unlock(); + } + } + + @Override + public void appendPendingClosure(final Closure closure) { + this.lock.lock(); + try { + this.queue.add(closure); + } finally { + this.lock.unlock(); + } + } + + @Override + public long popClosureUntil(final long endIndex, final List closures) { + return popClosureUntil(endIndex, closures, null); + } + + @Override + public long popClosureUntil(final long endIndex, final List closures, final List taskClosures) { + closures.clear(); + if (taskClosures != null) { + taskClosures.clear(); + } + this.lock.lock(); + try { + final int queueSize = this.queue.size(); + if (queueSize == 0 || endIndex < this.firstIndex) { + return endIndex + 1; + } + if (endIndex > this.firstIndex + queueSize - 1) { + LOG.error("Invalid endIndex={}, firstIndex={}, closureQueueSize={}", endIndex, this.firstIndex, + queueSize); + return -1; + } + final long outFirstIndex = this.firstIndex; + for (long i = outFirstIndex; i <= endIndex; i++) { + final Closure closure = this.queue.pollFirst(); + if (taskClosures != null && closure instanceof TaskClosure) { + taskClosures.add((TaskClosure) closure); + } + closures.add(closure); + } + this.firstIndex = endIndex + 1; + return outFirstIndex; + } finally { + this.lock.unlock(); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/JoinableClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/JoinableClosure.java new file mode 100644 index 0000000..2f18a4e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/JoinableClosure.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.util.Requires; + +/** + * @author jiachun.fjc + */ +public class JoinableClosure implements Closure { + + private final CountDownLatch latch = new CountDownLatch(1); + private final Closure closure; + + public JoinableClosure(Closure closure) { + this.closure = Requires.requireNonNull(closure, "closure"); + } + + @Override + public void run(final Status status) { + try { + this.closure.run(status); + } finally { + latch.countDown(); + } + } + + public void join() throws InterruptedException { + this.latch.await(); + } + + public void join(final long timeoutMillis) throws InterruptedException, TimeoutException { + if (!this.latch.await(timeoutMillis, TimeUnit.MILLISECONDS)) { + throw new TimeoutException("joined timeout"); + } + } + + public Closure getClosure() { + return closure; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/LoadSnapshotClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/LoadSnapshotClosure.java new file mode 100644 index 0000000..496d1df --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/LoadSnapshotClosure.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; + +/** + * Load snapshot closure + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:20:09 PM + */ +public interface LoadSnapshotClosure extends Closure { + + /** + * Start to load snapshot, returns a snapshot reader. + * + * @return a snapshot reader. + */ + SnapshotReader start(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ReadIndexClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ReadIndexClosure.java new file mode 100644 index 0000000..9bd23c6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/ReadIndexClosure.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.alipay.sofa.jraft.util.timer.Timeout; +import com.alipay.sofa.jraft.util.timer.Timer; +import com.alipay.sofa.jraft.util.timer.TimerTask; + +/** + * Read index closure + * + * @author dennis + */ +public abstract class ReadIndexClosure implements Closure { + + private static final Logger LOG = LoggerFactory + .getLogger(ReadIndexClosure.class); + + private static final AtomicIntegerFieldUpdater STATE_UPDATER = AtomicIntegerFieldUpdater + .newUpdater( + ReadIndexClosure.class, + "state"); + + private static final long DEFAULT_TIMEOUT = SystemPropertyUtil.getInt( + "jraft.read-index.timeout", + 2 * 1000); + + private static final int PENDING = 0; + private static final int COMPLETE = 1; + private static final int TIMEOUT = 2; + + /** + * Invalid log index -1. + */ + public static final long INVALID_LOG_INDEX = -1; + + private long index = INVALID_LOG_INDEX; + private byte[] requestContext; + + private volatile int state = PENDING; + + public ReadIndexClosure() { + this(DEFAULT_TIMEOUT); + } + + /** + * Create a read-index closure with a timeout parameter. + * + * @param timeoutMs timeout millis + */ + public ReadIndexClosure(long timeoutMs) { + if (timeoutMs >= 0) { + // Lazy to init the timer + TimeoutScanner.TIMER.newTimeout(new TimeoutTask(this), timeoutMs, TimeUnit.MILLISECONDS); + } + } + + /** + * Called when ReadIndex can be executed. + * + * @param status the readIndex status. + * @param index the committed index when starts readIndex. + * @param reqCtx the request context passed by {@link Node#readIndex(byte[], ReadIndexClosure)}. + * @see Node#readIndex(byte[], ReadIndexClosure) + */ + public abstract void run(final Status status, final long index, final byte[] reqCtx); + + /** + * Set callback result, called by jraft. + * + * @param index the committed index. + * @param reqCtx the request context passed by {@link Node#readIndex(byte[], ReadIndexClosure)}. + */ + public void setResult(final long index, final byte[] reqCtx) { + this.index = index; + this.requestContext = reqCtx; + } + + /** + * The committed log index when starts readIndex request. return -1 if fails. + * + * @return returns the committed index. returns -1 if fails. + */ + public long getIndex() { + return this.index; + } + + /** + * Returns the request context. + * + * @return the request context. + */ + public byte[] getRequestContext() { + return this.requestContext; + } + + @Override + public void run(final Status status) { + if (!STATE_UPDATER.compareAndSet(this, PENDING, COMPLETE)) { + LOG.warn("A timeout read-index response finally returned: {}.", status); + return; + } + + try { + run(status, this.index, this.requestContext); + } catch (final Throwable t) { + LOG.error("Fail to run ReadIndexClosure with status: {}.", status, t); + } + } + + static class TimeoutTask implements TimerTask { + + private final ReadIndexClosure closure; + + TimeoutTask(ReadIndexClosure closure) { + this.closure = closure; + } + + @Override + public void run(final Timeout timeout) throws Exception { + if (!STATE_UPDATER.compareAndSet(this.closure, PENDING, TIMEOUT)) { + return; + } + + final Status status = new Status(RaftError.ETIMEDOUT, "read-index request timeout"); + try { + this.closure.run(status, INVALID_LOG_INDEX, null); + } catch (final Throwable t) { + LOG.error("[Timeout] fail to run ReadIndexClosure with status: {}.", status, t); + } + } + } + + /** + * Lazy to create a timer + */ + static class TimeoutScanner { + private static final Timer TIMER = JRaftUtils.raftTimerFactory().createTimer("read-index.timeout.scanner"); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SaveSnapshotClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SaveSnapshotClosure.java new file mode 100644 index 0000000..4b1b9b6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SaveSnapshotClosure.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; + +/** + * Save snapshot closure + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:21:30 PM + */ +public interface SaveSnapshotClosure extends Closure { + + /** + * Starts to save snapshot, returns the writer. + * + * @param meta metadata of snapshot. + * @return returns snapshot writer. + */ + SnapshotWriter start(final SnapshotMeta meta); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SynchronizedClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SynchronizedClosure.java new file mode 100644 index 0000000..c1d2c2a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/SynchronizedClosure.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import java.util.concurrent.CountDownLatch; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; + +/** + * A special Closure which provides synchronization primitives. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-16 2:45:34 PM + */ +public class SynchronizedClosure implements Closure { + + private CountDownLatch latch; + private volatile Status status; + /** + * Latch count to reset + */ + private int count; + + public SynchronizedClosure() { + this(1); + } + + public SynchronizedClosure(final int n) { + this.count = n; + this.latch = new CountDownLatch(n); + } + + /** + * Get last ran status + * + * @return returns the last ran status + */ + public Status getStatus() { + return this.status; + } + + @Override + public void run(final Status status) { + this.status = status; + this.latch.countDown(); + } + + /** + * Wait for closure run + * + * @return status + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + public Status await() throws InterruptedException { + this.latch.await(); + return this.status; + } + + /** + * Reset the closure + */ + public void reset() { + this.status = null; + this.latch = new CountDownLatch(this.count); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/TaskClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/TaskClosure.java new file mode 100644 index 0000000..23c1415 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/closure/TaskClosure.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.closure; + +import com.alipay.sofa.jraft.Closure; + +/** + * Closure for task applying. + * @author dennis + */ +public interface TaskClosure extends Closure { + + /** + * Called when task is committed to majority peers of the + * RAFT group but before it is applied to state machine. + * + * Note: user implementation should not block + * this method and throw any exceptions. + */ + void onCommitted(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/Configuration.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/Configuration.java new file mode 100644 index 0000000..7175980 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/Configuration.java @@ -0,0 +1,324 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.conf; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.util.Copiable; +import com.alipay.sofa.jraft.util.Requires; + +/** + * A configuration with a set of peers. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-15 11:00:26 AM + */ +public class Configuration implements Iterable, Copiable { + + private static final Logger LOG = LoggerFactory.getLogger(Configuration.class); + + private static final String LEARNER_POSTFIX = "/learner"; + + private List peers = new ArrayList<>(); + + // use LinkedHashSet to keep insertion order. + private LinkedHashSet learners = new LinkedHashSet<>(); + + public Configuration() { + super(); + } + + /** + * Construct a configuration instance with peers. + * + * @param conf configuration + */ + public Configuration(final Iterable conf) { + this(conf, null); + } + + /** + * Construct a configuration from another conf. + * + * @param conf configuration + */ + public Configuration(final Configuration conf) { + this(conf.getPeers(), conf.getLearners()); + } + + /** + * Construct a Configuration instance with peers and learners. + * + * @param conf peers configuration + * @param learners learners + * @since 1.3.0 + */ + public Configuration(final Iterable conf, final Iterable learners) { + Requires.requireNonNull(conf, "conf"); + for (final PeerId peer : conf) { + this.peers.add(peer.copy()); + } + addLearners(learners); + } + + public void setLearners(final LinkedHashSet learners) { + this.learners = learners; + } + + /** + * Add a learner peer. + * + * @param learner learner to add + * @return true when add successfully. + */ + public boolean addLearner(final PeerId learner) { + return this.learners.add(learner); + } + + /** + * Add learners in batch, returns the added count. + * + * @param learners learners to add + * @return the total added count + */ + public int addLearners(final Iterable learners) { + int ret = 0; + if (learners != null) { + for (final PeerId peer : learners) { + if (this.learners.add(peer.copy())) { + ret++; + } + } + } + return ret; + } + + /** + * Remove a learner peer. + * + * @param learner learner to remove + * @return true when remove successfully. + */ + public boolean removeLearner(final PeerId learner) { + return this.learners.remove(learner); + } + + /** + * Retrieve the learners set. + * + * @return learners + */ + public LinkedHashSet getLearners() { + return this.learners; + } + + /** + * Retrieve the learners set copy. + * + * @return learners + */ + public List listLearners() { + return new ArrayList<>(this.learners); + } + + @Override + public Configuration copy() { + return new Configuration(this.peers, this.learners); + } + + /** + * Returns true when the configuration is valid. + * + * @return true if the configuration is valid. + */ + public boolean isValid() { + final Set intersection = new HashSet<>(this.peers); + intersection.retainAll(this.learners); + return !this.peers.isEmpty() && intersection.isEmpty(); + } + + public void reset() { + this.peers.clear(); + this.learners.clear(); + } + + public boolean isEmpty() { + return this.peers.isEmpty(); + } + + /** + * Returns the peers total number. + * + * @return total num of peers + */ + public int size() { + return this.peers.size(); + } + + @Override + public Iterator iterator() { + return this.peers.iterator(); + } + + public Set getPeerSet() { + return new HashSet<>(this.peers); + } + + public List listPeers() { + return new ArrayList<>(this.peers); + } + + public List getPeers() { + return this.peers; + } + + public void setPeers(final List peers) { + this.peers.clear(); + for (final PeerId peer : peers) { + this.peers.add(peer.copy()); + } + } + + public void appendPeers(final Collection set) { + this.peers.addAll(set); + } + + public boolean addPeer(final PeerId peer) { + return this.peers.add(peer); + } + + public boolean removePeer(final PeerId peer) { + return this.peers.remove(peer); + } + + public boolean contains(final PeerId peer) { + return this.peers.contains(peer); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((this.learners == null) ? 0 : this.learners.hashCode()); + result = prime * result + ((this.peers == null) ? 0 : this.peers.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + Configuration other = (Configuration) obj; + if (this.learners == null) { + if (other.learners != null) { + return false; + } + } else if (!this.learners.equals(other.learners)) { + return false; + } + if (this.peers == null) { + return other.peers == null; + } else { + return this.peers.equals(other.peers); + } + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder(); + final List peers = listPeers(); + int i = 0; + int size = peers.size(); + for (final PeerId peer : peers) { + sb.append(peer); + if (i < size - 1 || !this.learners.isEmpty()) { + sb.append(","); + } + i++; + } + + size = this.learners.size(); + i = 0; + for (final PeerId peer : this.learners) { + sb.append(peer).append(LEARNER_POSTFIX); + if (i < size - 1) { + sb.append(","); + } + i++; + } + + return sb.toString(); + } + + public boolean parse(final String conf) { + if (StringUtils.isBlank(conf)) { + return false; + } + reset(); + final String[] peerStrs = StringUtils.split(conf, ','); + for (String peerStr : peerStrs) { + final PeerId peer = new PeerId(); + int index; + boolean isLearner = false; + if ((index = peerStr.indexOf(LEARNER_POSTFIX)) > 0) { + // It's a learner + peerStr = peerStr.substring(0, index); + isLearner = true; + } + if (peer.parse(peerStr)) { + if (isLearner) { + addLearner(peer); + } else { + addPeer(peer); + } + } else { + LOG.error("Fail to parse peer {} in {}, ignore it.", peerStr, conf); + } + } + return true; + } + + /** + * Get the difference between |*this| and |rhs| + * |included| would be assigned to |*this| - |rhs| + * |excluded| would be assigned to |rhs| - |*this| + */ + public void diff(final Configuration rhs, final Configuration included, final Configuration excluded) { + included.peers = new ArrayList<>(this.peers); + included.peers.removeAll(rhs.peers); + excluded.peers = new ArrayList<>(rhs.peers); + excluded.peers.removeAll(this.peers); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationEntry.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationEntry.java new file mode 100644 index 0000000..d13fd17 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationEntry.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.conf; + +import java.util.HashSet; +import java.util.Set; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; + +/** + * A configuration entry with current peers and old peers. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:25:06 PM + */ +public class ConfigurationEntry { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigurationEntry.class); + + private LogId id = new LogId(0, 0); + private Configuration conf = new Configuration(); + private Configuration oldConf = new Configuration(); + + public LogId getId() { + return this.id; + } + + public void setId(final LogId id) { + this.id = id; + } + + public Configuration getConf() { + return this.conf; + } + + public void setConf(final Configuration conf) { + this.conf = conf; + } + + public Configuration getOldConf() { + return this.oldConf; + } + + public void setOldConf(final Configuration oldConf) { + this.oldConf = oldConf; + } + + public ConfigurationEntry() { + super(); + } + + public ConfigurationEntry(final LogId id, final Configuration conf, final Configuration oldConf) { + super(); + this.id = id; + this.conf = conf; + this.oldConf = oldConf; + } + + public boolean isStable() { + return this.oldConf.isEmpty(); + } + + public boolean isEmpty() { + return this.conf.isEmpty(); + } + + public Set listPeers() { + final Set ret = new HashSet<>(this.conf.listPeers()); + ret.addAll(this.oldConf.listPeers()); + return ret; + } + + /** + * Returns true when the conf entry is valid. + * + * @return if the the entry is valid + */ + public boolean isValid() { + if (!this.conf.isValid()) { + return false; + } + + // The peer set and learner set should not have intersection set. + final Set intersection = listPeers(); + intersection.retainAll(listLearners()); + if (intersection.isEmpty()) { + return true; + } + LOG.error("Invalid conf entry {}, peers and learners have intersection: {}.", this, intersection); + return false; + } + + public Set listLearners() { + final Set ret = new HashSet<>(this.conf.getLearners()); + ret.addAll(this.oldConf.getLearners()); + return ret; + } + + public boolean containsLearner(final PeerId learner) { + return this.conf.getLearners().contains(learner) || this.oldConf.getLearners().contains(learner); + } + + public boolean contains(final PeerId peer) { + return this.conf.contains(peer) || this.oldConf.contains(peer); + } + + @Override + public String toString() { + return "ConfigurationEntry [id=" + this.id + ", conf=" + this.conf + ", oldConf=" + this.oldConf + "]"; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationManager.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationManager.java new file mode 100644 index 0000000..3337699 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/conf/ConfigurationManager.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.conf; + +import java.util.LinkedList; +import java.util.ListIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.util.Requires; + +/** + * Configuration manager + * + * @author boyan (boyan@alibaba-inc.com) + *

+ * 2018-Apr-04 2:24:54 PM + */ +public class ConfigurationManager { + + private static final Logger LOG = LoggerFactory.getLogger(ConfigurationManager.class); + + private final LinkedList configurations = new LinkedList<>(); + private ConfigurationEntry snapshot = new ConfigurationEntry(); + + /** + * Adds a new conf entry. + */ + public boolean add(final ConfigurationEntry entry) { + if (!this.configurations.isEmpty()) { + if (this.configurations.peekLast().getId().getIndex() >= entry.getId().getIndex()) { + LOG.error("Did you forget to call truncateSuffix before the last log index goes back."); + return false; + } + } + return this.configurations.add(entry); + } + + /** + * [1, first_index_kept) are being discarded + */ + public void truncatePrefix(final long firstIndexKept) { + while (!this.configurations.isEmpty() && this.configurations.peekFirst().getId().getIndex() < firstIndexKept) { + this.configurations.pollFirst(); + } + } + + /** + * (last_index_kept, infinity) are being discarded + */ + public void truncateSuffix(final long lastIndexKept) { + while (!this.configurations.isEmpty() && this.configurations.peekLast().getId().getIndex() > lastIndexKept) { + this.configurations.pollLast(); + } + } + + public ConfigurationEntry getSnapshot() { + return this.snapshot; + } + + public void setSnapshot(final ConfigurationEntry snapshot) { + this.snapshot = snapshot; + } + + public ConfigurationEntry getLastConfiguration() { + if (this.configurations.isEmpty()) { + return snapshot; + } else { + return this.configurations.peekLast(); + } + } + + public ConfigurationEntry get(final long lastIncludedIndex) { + if (this.configurations.isEmpty()) { + Requires.requireTrue(lastIncludedIndex >= this.snapshot.getId().getIndex(), + "lastIncludedIndex %d is less than snapshot index %d", lastIncludedIndex, this.snapshot.getId() + .getIndex()); + return this.snapshot; + } + ListIterator it = this.configurations.listIterator(); + while (it.hasNext()) { + if (it.next().getId().getIndex() > lastIncludedIndex) { + it.previous(); + break; + } + } + if (it.hasPrevious()) { + // find the first position that is less than or equal to lastIncludedIndex. + return it.previous(); + } else { + // position not found position, return snapshot. + return this.snapshot; + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/BallotBox.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/BallotBox.java new file mode 100644 index 0000000..64d2bf9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/BallotBox.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.concurrent.locks.StampedLock; + +import javax.annotation.concurrent.ThreadSafe; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.closure.ClosureQueue; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.Ballot; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.BallotBoxOptions; +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.SegmentList; + +/** + * Ballot box for voting. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:32:10 PM + */ +@ThreadSafe +public class BallotBox implements Lifecycle, Describer { + + private static final Logger LOG = LoggerFactory.getLogger(BallotBox.class); + + private FSMCaller waiter; + private ClosureQueue closureQueue; + private final StampedLock stampedLock = new StampedLock(); + private long lastCommittedIndex = 0; + private long pendingIndex; + private final SegmentList pendingMetaQueue = new SegmentList<>(false); + + @OnlyForTest + long getPendingIndex() { + return this.pendingIndex; + } + + @OnlyForTest + SegmentList getPendingMetaQueue() { + return this.pendingMetaQueue; + } + + public long getLastCommittedIndex() { + long stamp = this.stampedLock.tryOptimisticRead(); + final long optimisticVal = this.lastCommittedIndex; + if (this.stampedLock.validate(stamp)) { + return optimisticVal; + } + stamp = this.stampedLock.readLock(); + try { + return this.lastCommittedIndex; + } finally { + this.stampedLock.unlockRead(stamp); + } + } + + @Override + public boolean init(final BallotBoxOptions opts) { + if (opts.getWaiter() == null || opts.getClosureQueue() == null) { + LOG.error("waiter or closure queue is null."); + return false; + } + this.waiter = opts.getWaiter(); + this.closureQueue = opts.getClosureQueue(); + return true; + } + + /** + * Called by leader, otherwise the behavior is undefined + * Set logs in [first_log_index, last_log_index] are stable at |peer|. + */ + public boolean commitAt(final long firstLogIndex, final long lastLogIndex, final PeerId peer) { + // TODO use lock-free algorithm here? + final long stamp = this.stampedLock.writeLock(); + long lastCommittedIndex = 0; + try { + if (this.pendingIndex == 0) { + return false; + } + if (lastLogIndex < this.pendingIndex) { + return true; + } + + if (lastLogIndex >= this.pendingIndex + this.pendingMetaQueue.size()) { + throw new ArrayIndexOutOfBoundsException(); + } + + final long startAt = Math.max(this.pendingIndex, firstLogIndex); + Ballot.PosHint hint = new Ballot.PosHint(); + for (long logIndex = startAt; logIndex <= lastLogIndex; logIndex++) { + final Ballot bl = this.pendingMetaQueue.get((int) (logIndex - this.pendingIndex)); + hint = bl.grant(peer, hint); + if (bl.isGranted()) { + lastCommittedIndex = logIndex; + } + } + if (lastCommittedIndex == 0) { + return true; + } + // When removing a peer off the raft group which contains even number of + // peers, the quorum would decrease by 1, e.g. 3 of 4 changes to 2 of 3. In + // this case, the log after removal may be committed before some previous + // logs, since we use the new configuration to deal the quorum of the + // removal request, we think it's safe to commit all the uncommitted + // previous logs, which is not well proved right now + this.pendingMetaQueue.removeFromFirst((int) (lastCommittedIndex - this.pendingIndex) + 1); + LOG.debug("Committed log fromIndex={}, toIndex={}.", this.pendingIndex, lastCommittedIndex); + this.pendingIndex = lastCommittedIndex + 1; + this.lastCommittedIndex = lastCommittedIndex; + } finally { + this.stampedLock.unlockWrite(stamp); + } + this.waiter.onCommitted(lastCommittedIndex); + return true; + } + + /** + * Called when the leader steps down, otherwise the behavior is undefined + * When a leader steps down, the uncommitted user applications should + * fail immediately, which the new leader will deal whether to commit or + * truncate. + */ + public void clearPendingTasks() { + final long stamp = this.stampedLock.writeLock(); + try { + this.pendingMetaQueue.clear(); + this.pendingIndex = 0; + this.closureQueue.clear(); + } finally { + this.stampedLock.unlockWrite(stamp); + } + } + + /** + * Called when a candidate becomes the new leader, otherwise the behavior is + * undefined. + * According the the raft algorithm, the logs from previous terms can't be + * committed until a log at the new term becomes committed, so + * |newPendingIndex| should be |last_log_index| + 1. + * @param newPendingIndex pending index of new leader + * @return returns true if reset success + */ + public boolean resetPendingIndex(final long newPendingIndex) { + final long stamp = this.stampedLock.writeLock(); + try { + if (!(this.pendingIndex == 0 && this.pendingMetaQueue.isEmpty())) { + LOG.error("resetPendingIndex fail, pendingIndex={}, pendingMetaQueueSize={}.", this.pendingIndex, + this.pendingMetaQueue.size()); + return false; + } + if (newPendingIndex <= this.lastCommittedIndex) { + LOG.error("resetPendingIndex fail, newPendingIndex={}, lastCommittedIndex={}.", newPendingIndex, + this.lastCommittedIndex); + return false; + } + this.pendingIndex = newPendingIndex; + this.closureQueue.resetFirstIndex(newPendingIndex); + return true; + } finally { + this.stampedLock.unlockWrite(stamp); + } + } + + /** + * Called by leader, otherwise the behavior is undefined + * Store application context before replication. + * + * @param conf current configuration + * @param oldConf old configuration + * @param done callback + * @return returns true on success + */ + public boolean appendPendingTask(final Configuration conf, final Configuration oldConf, final Closure done) { + final Ballot bl = new Ballot(); + if (!bl.init(conf, oldConf)) { + LOG.error("Fail to init ballot."); + return false; + } + final long stamp = this.stampedLock.writeLock(); + try { + if (this.pendingIndex <= 0) { + LOG.error("Fail to appendingTask, pendingIndex={}.", this.pendingIndex); + return false; + } + this.pendingMetaQueue.add(bl); + this.closureQueue.appendPendingClosure(done); + return true; + } finally { + this.stampedLock.unlockWrite(stamp); + } + } + + /** + * Called by follower, otherwise the behavior is undefined. + * Set committed index received from leader + * + * @param lastCommittedIndex last committed index + * @return returns true if set success + */ + public boolean setLastCommittedIndex(final long lastCommittedIndex) { + boolean doUnlock = true; + final long stamp = this.stampedLock.writeLock(); + try { + if (this.pendingIndex != 0 || !this.pendingMetaQueue.isEmpty()) { + Requires.requireTrue(lastCommittedIndex < this.pendingIndex, + "Node changes to leader, pendingIndex=%d, param lastCommittedIndex=%d", this.pendingIndex, + lastCommittedIndex); + return false; + } + if (lastCommittedIndex < this.lastCommittedIndex) { + return false; + } + if (lastCommittedIndex > this.lastCommittedIndex) { + this.lastCommittedIndex = lastCommittedIndex; + this.stampedLock.unlockWrite(stamp); + doUnlock = false; + this.waiter.onCommitted(lastCommittedIndex); + } + } finally { + if (doUnlock) { + this.stampedLock.unlockWrite(stamp); + } + } + return true; + } + + @Override + public void shutdown() { + clearPendingTasks(); + } + + @Override + public void describe(final Printer out) { + long _lastCommittedIndex; + long _pendingIndex; + long _pendingMetaQueueSize; + long stamp = this.stampedLock.tryOptimisticRead(); + if (this.stampedLock.validate(stamp)) { + _lastCommittedIndex = this.lastCommittedIndex; + _pendingIndex = this.pendingIndex; + _pendingMetaQueueSize = this.pendingMetaQueue.size(); + } else { + stamp = this.stampedLock.readLock(); + try { + _lastCommittedIndex = this.lastCommittedIndex; + _pendingIndex = this.pendingIndex; + _pendingMetaQueueSize = this.pendingMetaQueue.size(); + } finally { + this.stampedLock.unlockRead(stamp); + } + } + out.print(" lastCommittedIndex: ") // + .println(_lastCommittedIndex); + out.print(" pendingIndex: ") // + .println(_pendingIndex); + out.print(" pendingMetaQueueSize: ") // + .println(_pendingMetaQueueSize); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/CliServiceImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/CliServiceImpl.java new file mode 100644 index 0000000..3bc56b9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/CliServiceImpl.java @@ -0,0 +1,672 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.CliService; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.JRaftException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.rpc.CliClientService; +import com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse; +import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.Utils; +import com.google.protobuf.Message; +import com.google.protobuf.ProtocolStringList; + +/** + * Cli service implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class CliServiceImpl implements CliService { + + private static final Logger LOG = LoggerFactory.getLogger(CliServiceImpl.class); + + private CliOptions cliOptions; + private CliClientService cliClientService; + + @Override + public synchronized boolean init(final CliOptions opts) { + Requires.requireNonNull(opts, "Null cli options"); + + if (this.cliClientService != null) { + return true; + } + this.cliOptions = opts; + this.cliClientService = new CliClientServiceImpl(); + return this.cliClientService.init(this.cliOptions); + } + + @Override + public synchronized void shutdown() { + if (this.cliClientService == null) { + return; + } + this.cliClientService.shutdown(); + this.cliClientService = null; + } + + private void recordConfigurationChange(final String groupId, final List oldPeersList, + final List newPeersList) { + final Configuration oldConf = new Configuration(); + for (final String peerIdStr : oldPeersList) { + final PeerId oldPeer = new PeerId(); + oldPeer.parse(peerIdStr); + oldConf.addPeer(oldPeer); + } + final Configuration newConf = new Configuration(); + for (final String peerIdStr : newPeersList) { + final PeerId newPeer = new PeerId(); + newPeer.parse(peerIdStr); + newConf.addPeer(newPeer); + } + LOG.info("Configuration of replication group {} changed from {} to {}.", groupId, oldConf, newConf); + } + + private Status checkLeaderAndConnect(final String groupId, final Configuration conf, final PeerId leaderId) { + final Status st = getLeader(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + if (!this.cliClientService.connect(leaderId.getEndpoint())) { + return new Status(-1, "Fail to init channel to leader %s", leaderId); + } + + return Status.OK(); + } + + @Override + public Status addPeer(final String groupId, final Configuration conf, final PeerId peer) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireNonNull(peer, "Null peer"); + + final PeerId leaderId = new PeerId(); + final Status st = checkLeaderAndConnect(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + final AddPeerRequest.Builder rb = AddPeerRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()) // + .setPeerId(peer.toString()); + + try { + final Message result = this.cliClientService.addPeer(leaderId.getEndpoint(), rb.build(), null).get(); + if (result instanceof AddPeerResponse) { + final AddPeerResponse resp = (AddPeerResponse) result; + recordConfigurationChange(groupId, resp.getOldPeersList(), resp.getNewPeersList()); + return Status.OK(); + } else { + return statusFromResponse(result); + } + + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + private Status statusFromResponse(final Message result) { + final ErrorResponse resp = (ErrorResponse) result; + return new Status(resp.getErrorCode(), resp.getErrorMsg()); + } + + @Override + public Status removePeer(final String groupId, final Configuration conf, final PeerId peer) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireNonNull(peer, "Null peer"); + Requires.requireTrue(!peer.isEmpty(), "Removing peer is blank"); + + final PeerId leaderId = new PeerId(); + final Status st = checkLeaderAndConnect(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + final RemovePeerRequest.Builder rb = RemovePeerRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()) // + .setPeerId(peer.toString()); + + try { + final Message result = this.cliClientService.removePeer(leaderId.getEndpoint(), rb.build(), null).get(); + if (result instanceof RemovePeerResponse) { + final RemovePeerResponse resp = (RemovePeerResponse) result; + recordConfigurationChange(groupId, resp.getOldPeersList(), resp.getNewPeersList()); + return Status.OK(); + } else { + return statusFromResponse(result); + + } + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status changePeers(final String groupId, final Configuration conf, final Configuration newPeers) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireNonNull(newPeers, "Null new peers"); + + final PeerId leaderId = new PeerId(); + final Status st = checkLeaderAndConnect(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + final ChangePeersRequest.Builder rb = ChangePeersRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()); + for (final PeerId peer : newPeers) { + rb.addNewPeers(peer.toString()); + } + + try { + final Message result = this.cliClientService.changePeers(leaderId.getEndpoint(), rb.build(), null).get(); + if (result instanceof ChangePeersResponse) { + final ChangePeersResponse resp = (ChangePeersResponse) result; + recordConfigurationChange(groupId, resp.getOldPeersList(), resp.getNewPeersList()); + return Status.OK(); + } else { + return statusFromResponse(result); + + } + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status resetPeer(final String groupId, final PeerId peerId, final Configuration newPeers) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(peerId, "Null peerId"); + Requires.requireNonNull(newPeers, "Null new peers"); + + if (!this.cliClientService.connect(peerId.getEndpoint())) { + return new Status(-1, "Fail to init channel to %s", peerId); + } + + final ResetPeerRequest.Builder rb = ResetPeerRequest.newBuilder() // + .setGroupId(groupId) // + .setPeerId(peerId.toString()); + for (final PeerId peer : newPeers) { + rb.addNewPeers(peer.toString()); + } + + try { + final Message result = this.cliClientService.resetPeer(peerId.getEndpoint(), rb.build(), null).get(); + return statusFromResponse(result); + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + private void checkPeers(final Collection peers) { + for (final PeerId peer : peers) { + Requires.requireNonNull(peer, "Null peer in collection"); + } + } + + @Override + public Status addLearners(final String groupId, final Configuration conf, final List learners) { + checkLearnersOpParams(groupId, conf, learners); + + final PeerId leaderId = new PeerId(); + final Status st = getLeader(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + if (!this.cliClientService.connect(leaderId.getEndpoint())) { + return new Status(-1, "Fail to init channel to leader %s", leaderId); + } + final AddLearnersRequest.Builder rb = AddLearnersRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()); + for (final PeerId peer : learners) { + rb.addLearners(peer.toString()); + } + + try { + final Message result = this.cliClientService.addLearners(leaderId.getEndpoint(), rb.build(), null).get(); + return processLearnersOpResponse(groupId, result, "adding learners: %s", learners); + + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + private void checkLearnersOpParams(final String groupId, final Configuration conf, final List learners) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireTrue(learners != null && !learners.isEmpty(), "Empty peers"); + checkPeers(learners); + } + + private Status processLearnersOpResponse(final String groupId, final Message result, final String fmt, + final Object... formatArgs) { + if (result instanceof LearnersOpResponse) { + final LearnersOpResponse resp = (LearnersOpResponse) result; + final Configuration oldConf = new Configuration(); + for (final String peerIdStr : resp.getOldLearnersList()) { + final PeerId oldPeer = new PeerId(); + oldPeer.parse(peerIdStr); + oldConf.addLearner(oldPeer); + } + final Configuration newConf = new Configuration(); + for (final String peerIdStr : resp.getNewLearnersList()) { + final PeerId newPeer = new PeerId(); + newPeer.parse(peerIdStr); + newConf.addLearner(newPeer); + } + + LOG.info("Learners of replication group {} changed from {} to {} after {}.", groupId, oldConf, newConf, + String.format(fmt, formatArgs)); + return Status.OK(); + } else { + return statusFromResponse(result); + } + } + + @Override + public Status removeLearners(final String groupId, final Configuration conf, final List learners) { + checkLearnersOpParams(groupId, conf, learners); + + final PeerId leaderId = new PeerId(); + final Status st = getLeader(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + if (!this.cliClientService.connect(leaderId.getEndpoint())) { + return new Status(-1, "Fail to init channel to leader %s", leaderId); + } + final RemoveLearnersRequest.Builder rb = RemoveLearnersRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()); + for (final PeerId peer : learners) { + rb.addLearners(peer.toString()); + } + + try { + final Message result = this.cliClientService.removeLearners(leaderId.getEndpoint(), rb.build(), null).get(); + return processLearnersOpResponse(groupId, result, "removing learners: %s", learners); + + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status learner2Follower(final String groupId, final Configuration conf, final PeerId learner) { + Status status = removeLearners(groupId, conf, Arrays.asList(learner)); + if (status.isOk()) { + status = addPeer(groupId, conf, new PeerId(learner.getIp(), learner.getPort())); + } + return status; + } + + @Override + public Status resetLearners(final String groupId, final Configuration conf, final List learners) { + checkLearnersOpParams(groupId, conf, learners); + + final PeerId leaderId = new PeerId(); + final Status st = getLeader(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + if (!this.cliClientService.connect(leaderId.getEndpoint())) { + return new Status(-1, "Fail to init channel to leader %s", leaderId); + } + final ResetLearnersRequest.Builder rb = ResetLearnersRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()); + for (final PeerId peer : learners) { + rb.addLearners(peer.toString()); + } + + try { + final Message result = this.cliClientService.resetLearners(leaderId.getEndpoint(), rb.build(), null).get(); + return processLearnersOpResponse(groupId, result, "resetting learners: %s", learners); + + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireNonNull(peer, "Null peer"); + + final PeerId leaderId = new PeerId(); + final Status st = checkLeaderAndConnect(groupId, conf, leaderId); + if (!st.isOk()) { + return st; + } + + final TransferLeaderRequest.Builder rb = TransferLeaderRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()); + if (!peer.isEmpty()) { + rb.setPeerId(peer.toString()); + } + + try { + final Message result = this.cliClientService.transferLeader(leaderId.getEndpoint(), rb.build(), null).get(); + return statusFromResponse(result); + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status snapshot(final String groupId, final PeerId peer) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(peer, "Null peer"); + + if (!this.cliClientService.connect(peer.getEndpoint())) { + return new Status(-1, "Fail to init channel to %s", peer); + } + + final SnapshotRequest.Builder rb = SnapshotRequest.newBuilder() // + .setGroupId(groupId) // + .setPeerId(peer.toString()); + + try { + final Message result = this.cliClientService.snapshot(peer.getEndpoint(), rb.build(), null).get(); + return statusFromResponse(result); + } catch (final Exception e) { + return new Status(-1, e.getMessage()); + } + } + + @Override + public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(leaderId, "Null leader id"); + + if (conf == null || conf.isEmpty()) { + return new Status(RaftError.EINVAL, "Empty group configuration"); + } + + final Status st = new Status(-1, "Fail to get leader of group %s", groupId); + for (final PeerId peer : conf) { + if (!this.cliClientService.connect(peer.getEndpoint())) { + LOG.error("Fail to connect peer {} to get leader for group {}.", peer, groupId); + continue; + } + + final GetLeaderRequest.Builder rb = GetLeaderRequest.newBuilder() // + .setGroupId(groupId) // + .setPeerId(peer.toString()); + + final Future result = this.cliClientService.getLeader(peer.getEndpoint(), rb.build(), null); + try { + + final Message msg = result.get( + this.cliOptions.getTimeoutMs() <= 0 ? this.cliOptions.getRpcDefaultTimeout() : this.cliOptions + .getTimeoutMs(), TimeUnit.MILLISECONDS); + if (msg instanceof ErrorResponse) { + if (st.isOk()) { + st.setError(-1, ((ErrorResponse) msg).getErrorMsg()); + } else { + final String savedMsg = st.getErrorMsg(); + st.setError(-1, "%s, %s", savedMsg, ((ErrorResponse) msg).getErrorMsg()); + } + } else { + final GetLeaderResponse response = (GetLeaderResponse) msg; + if (leaderId.parse(response.getLeaderId())) { + break; + } + } + } catch (final Exception e) { + if (st.isOk()) { + st.setError(-1, e.getMessage()); + } else { + final String savedMsg = st.getErrorMsg(); + st.setError(-1, "%s, %s", savedMsg, e.getMessage()); + } + } + } + + if (leaderId.isEmpty()) { + return st; + } + return Status.OK(); + } + + @Override + public List getPeers(final String groupId, final Configuration conf) { + return getPeers(groupId, conf, false, false); + } + + @Override + public List getAlivePeers(final String groupId, final Configuration conf) { + return getPeers(groupId, conf, false, true); + } + + @Override + public List getLearners(final String groupId, final Configuration conf) { + return getPeers(groupId, conf, true, false); + } + + @Override + public List getAliveLearners(final String groupId, final Configuration conf) { + return getPeers(groupId, conf, true, true); + } + + @Override + public Status rebalance(final Set balanceGroupIds, final Configuration conf, + final Map rebalancedLeaderIds) { + Requires.requireNonNull(balanceGroupIds, "Null balance group ids"); + Requires.requireTrue(!balanceGroupIds.isEmpty(), "Empty balance group ids"); + Requires.requireNonNull(conf, "Null configuration"); + Requires.requireTrue(!conf.isEmpty(), "No peers of configuration"); + + LOG.info("Rebalance start with raft groups={}.", balanceGroupIds); + + final long start = Utils.monotonicMs(); + int transfers = 0; + Status failedStatus = null; + final Queue groupDeque = new ArrayDeque<>(balanceGroupIds); + final LeaderCounter leaderCounter = new LeaderCounter(balanceGroupIds.size(), conf.size()); + for (;;) { + final String groupId = groupDeque.poll(); + if (groupId == null) { // well done + break; + } + + final PeerId leaderId = new PeerId(); + final Status leaderStatus = getLeader(groupId, conf, leaderId); + if (!leaderStatus.isOk()) { + failedStatus = leaderStatus; + break; + } + + if (rebalancedLeaderIds != null) { + rebalancedLeaderIds.put(groupId, leaderId); + } + + if (leaderCounter.incrementAndGet(leaderId) <= leaderCounter.getExpectedAverage()) { + // The num of leaders is less than the expected average, we are going to deal with others + continue; + } + + // Find the target peer and try to transfer the leader to this peer + final PeerId targetPeer = findTargetPeer(leaderId, groupId, conf, leaderCounter); + if (!targetPeer.isEmpty()) { + final Status transferStatus = transferLeader(groupId, conf, targetPeer); + transfers++; + if (!transferStatus.isOk()) { + // The failure of `transfer leader` usually means the node is busy, + // so we return failure status and should try `rebalance` again later. + failedStatus = transferStatus; + break; + } + + LOG.info("Group {} transfer leader to {}.", groupId, targetPeer); + leaderCounter.decrementAndGet(leaderId); + groupDeque.add(groupId); + if (rebalancedLeaderIds != null) { + rebalancedLeaderIds.put(groupId, targetPeer); + } + } + } + + final Status status = failedStatus != null ? failedStatus : Status.OK(); + if (LOG.isInfoEnabled()) { + LOG.info( + "Rebalanced raft groups={}, status={}, number of transfers={}, elapsed time={} ms, rebalanced result={}.", + balanceGroupIds, status, transfers, Utils.monotonicMs() - start, rebalancedLeaderIds); + } + return status; + } + + private PeerId findTargetPeer(final PeerId self, final String groupId, final Configuration conf, + final LeaderCounter leaderCounter) { + for (final PeerId peerId : getAlivePeers(groupId, conf)) { + if (peerId.equals(self)) { + continue; + } + if (leaderCounter.get(peerId) >= leaderCounter.getExpectedAverage()) { + continue; + } + return peerId; + } + return PeerId.emptyPeer(); + } + + private List getPeers(final String groupId, final Configuration conf, final boolean returnLearners, + final boolean onlyGetAlive) { + Requires.requireTrue(!StringUtils.isBlank(groupId), "Blank group id"); + Requires.requireNonNull(conf, "Null conf"); + + final PeerId leaderId = new PeerId(); + final Status st = getLeader(groupId, conf, leaderId); + if (!st.isOk()) { + throw new IllegalStateException(st.getErrorMsg()); + } + + if (!this.cliClientService.connect(leaderId.getEndpoint())) { + throw new IllegalStateException("Fail to init channel to leader " + leaderId); + } + + final GetPeersRequest.Builder rb = GetPeersRequest.newBuilder() // + .setGroupId(groupId) // + .setLeaderId(leaderId.toString()) // + .setOnlyAlive(onlyGetAlive); + + try { + final Message result = this.cliClientService.getPeers(leaderId.getEndpoint(), rb.build(), null).get( + this.cliOptions.getTimeoutMs() <= 0 ? this.cliOptions.getRpcDefaultTimeout() + : this.cliOptions.getTimeoutMs(), TimeUnit.MILLISECONDS); + if (result instanceof GetPeersResponse) { + final GetPeersResponse resp = (GetPeersResponse) result; + final List peerIdList = new ArrayList<>(); + final ProtocolStringList responsePeers = returnLearners ? resp.getLearnersList() : resp.getPeersList(); + for (final String peerIdStr : responsePeers) { + final PeerId newPeer = new PeerId(); + newPeer.parse(peerIdStr); + peerIdList.add(newPeer); + } + return peerIdList; + } else { + final ErrorResponse resp = (ErrorResponse) result; + throw new JRaftException(resp.getErrorMsg()); + } + } catch (final JRaftException e) { + throw e; + } catch (final Exception e) { + throw new JRaftException(e); + } + } + + public CliClientService getCliClientService() { + return this.cliClientService; + } + + private static class LeaderCounter { + + private final Map counter = new HashMap<>(); + // The expected average leader number on every peerId + private final int expectedAverage; + + public LeaderCounter(final int groupCount, final int peerCount) { + this.expectedAverage = (int) Math.ceil((double) groupCount / peerCount); + } + + public int getExpectedAverage() { + return this.expectedAverage; + } + + public int incrementAndGet(final PeerId peerId) { + return this.counter.compute(peerId, (ignored, num) -> num == null ? 1 : num + 1); + } + + public int decrementAndGet(final PeerId peerId) { + return this.counter.compute(peerId, (ignored, num) -> num == null ? 0 : num - 1); + } + + public int get(final PeerId peerId) { + return this.counter.getOrDefault(peerId, 0); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/DefaultJRaftServiceFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/DefaultJRaftServiceFactory.java new file mode 100644 index 0000000..26321f4 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/DefaultJRaftServiceFactory.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import org.apache.commons.lang.StringUtils; + +import com.alipay.sofa.jraft.JRaftServiceFactory; +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; +import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.LogStorage; +import com.alipay.sofa.jraft.storage.RaftMetaStorage; +import com.alipay.sofa.jraft.storage.SnapshotStorage; +import com.alipay.sofa.jraft.storage.impl.LocalRaftMetaStorage; +import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; +import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotStorage; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.SPI; + +/** + * The default factory for JRaft services. + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + * + */ +@SPI +public class DefaultJRaftServiceFactory implements JRaftServiceFactory { + + public static DefaultJRaftServiceFactory newInstance() { + return new DefaultJRaftServiceFactory(); + } + + @Override + public LogStorage createLogStorage(final String uri, final RaftOptions raftOptions) { + Requires.requireTrue(StringUtils.isNotBlank(uri), "Blank log storage uri."); + return new RocksDBLogStorage(uri, raftOptions); + } + + @Override + public SnapshotStorage createSnapshotStorage(final String uri, final RaftOptions raftOptions) { + Requires.requireTrue(!StringUtils.isBlank(uri), "Blank snapshot storage uri."); + return new LocalSnapshotStorage(uri, raftOptions); + } + + @Override + public RaftMetaStorage createRaftMetaStorage(final String uri, final RaftOptions raftOptions) { + Requires.requireTrue(!StringUtils.isBlank(uri), "Blank raft meta storage uri."); + return new LocalRaftMetaStorage(uri, raftOptions); + } + + @Override + public LogEntryCodecFactory createLogEntryCodecFactory() { + return LogEntryV2CodecFactory.getInstance(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ElectionPriority.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ElectionPriority.java new file mode 100644 index 0000000..16cae25 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ElectionPriority.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +/** + * ElectionPriority Type + * + * @author zongtanghu + */ +public class ElectionPriority { + + /** + * Priority -1 represents this node disabled the priority election function. + */ + public static final int Disabled = -1; + + /** + * Priority 0 is a special value so that a node will never participate in election. + */ + public static final int NotElected = 0; + + /** + * Priority 1 is a minimum value for priority election. + */ + public static final int MinValue = 1; +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/FSMCallerImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/FSMCallerImpl.java new file mode 100644 index 0000000..2f05340 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/FSMCallerImpl.java @@ -0,0 +1,726 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicLong; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.ClosureQueue; +import com.alipay.sofa.jraft.closure.LoadSnapshotClosure; +import com.alipay.sofa.jraft.closure.SaveSnapshotClosure; +import com.alipay.sofa.jraft.closure.TaskClosure; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.RaftOutter; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.FSMCallerOptions; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; +import com.alipay.sofa.jraft.util.DisruptorBuilder; +import com.alipay.sofa.jraft.util.DisruptorMetricSet; +import com.alipay.sofa.jraft.util.LogExceptionHandler; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.Utils; +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.EventTranslator; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +/** + * The finite state machine caller implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:12:14 AM + */ +public class FSMCallerImpl implements FSMCaller { + + private static final Logger LOG = LoggerFactory.getLogger(FSMCallerImpl.class); + + /** + * Task type + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:12:25 AM + */ + private enum TaskType { + IDLE, // + COMMITTED, // + SNAPSHOT_SAVE, // + SNAPSHOT_LOAD, // + LEADER_STOP, // + LEADER_START, // + START_FOLLOWING, // + STOP_FOLLOWING, // + SHUTDOWN, // + FLUSH, // + ERROR; + + private String metricName; + + public String metricName() { + if (this.metricName == null) { + this.metricName = "fsm-" + name().toLowerCase().replaceAll("_", "-"); + } + return this.metricName; + } + } + + /** + * Apply task for disruptor. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 11:12:35 AM + */ + private static class ApplyTask { + TaskType type; + // union fields + long committedIndex; + long term; + Status status; + LeaderChangeContext leaderChangeCtx; + Closure done; + CountDownLatch shutdownLatch; + + public void reset() { + this.type = null; + this.committedIndex = 0; + this.term = 0; + this.status = null; + this.leaderChangeCtx = null; + this.done = null; + this.shutdownLatch = null; + } + } + + private static class ApplyTaskFactory implements EventFactory { + + @Override + public ApplyTask newInstance() { + return new ApplyTask(); + } + } + + private class ApplyTaskHandler implements EventHandler { + // max committed index in current batch, reset to -1 every batch + private long maxCommittedIndex = -1; + + @Override + public void onEvent(final ApplyTask event, final long sequence, final boolean endOfBatch) throws Exception { + this.maxCommittedIndex = runApplyTask(event, this.maxCommittedIndex, endOfBatch); + } + } + + private LogManager logManager; + private StateMachine fsm; + private ClosureQueue closureQueue; + private final AtomicLong lastAppliedIndex; + private long lastAppliedTerm; + private Closure afterShutdown; + private NodeImpl node; + private volatile TaskType currTask; + private final AtomicLong applyingIndex; + private volatile RaftException error; + private Disruptor disruptor; + private RingBuffer taskQueue; + private volatile CountDownLatch shutdownLatch; + private NodeMetrics nodeMetrics; + private final CopyOnWriteArrayList lastAppliedLogIndexListeners = new CopyOnWriteArrayList<>(); + + public FSMCallerImpl() { + super(); + this.currTask = TaskType.IDLE; + this.lastAppliedIndex = new AtomicLong(0); + this.applyingIndex = new AtomicLong(0); + } + + @Override + public boolean init(final FSMCallerOptions opts) { + this.logManager = opts.getLogManager(); + this.fsm = opts.getFsm(); + this.closureQueue = opts.getClosureQueue(); + this.afterShutdown = opts.getAfterShutdown(); + this.node = opts.getNode(); + this.nodeMetrics = this.node.getNodeMetrics(); + this.lastAppliedIndex.set(opts.getBootstrapId().getIndex()); + notifyLastAppliedIndexUpdated(this.lastAppliedIndex.get()); + this.lastAppliedTerm = opts.getBootstrapId().getTerm(); + this.disruptor = DisruptorBuilder. newInstance() // + .setEventFactory(new ApplyTaskFactory()) // + .setRingBufferSize(opts.getDisruptorBufferSize()) // + .setThreadFactory(new NamedThreadFactory("JRaft-FSMCaller-Disruptor-", true)) // + .setProducerType(ProducerType.MULTI) // + .setWaitStrategy(new BlockingWaitStrategy()) // + .build(); + this.disruptor.handleEventsWith(new ApplyTaskHandler()); + this.disruptor.setDefaultExceptionHandler(new LogExceptionHandler(getClass().getSimpleName())); + this.taskQueue = this.disruptor.start(); + if (this.nodeMetrics.getMetricRegistry() != null) { + this.nodeMetrics.getMetricRegistry().register("jraft-fsm-caller-disruptor", + new DisruptorMetricSet(this.taskQueue)); + } + this.error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_NONE); + LOG.info("Starts FSMCaller successfully."); + return true; + } + + @Override + public synchronized void shutdown() { + if (this.shutdownLatch != null) { + return; + } + LOG.info("Shutting down FSMCaller..."); + + if (this.taskQueue != null) { + final CountDownLatch latch = new CountDownLatch(1); + this.shutdownLatch = latch; + Utils.runInThread(() -> this.taskQueue.publishEvent((task, sequence) -> { + task.reset(); + task.type = TaskType.SHUTDOWN; + task.shutdownLatch = latch; + })); + } + doShutdown(); + } + + @Override + public void addLastAppliedLogIndexListener(final LastAppliedLogIndexListener listener) { + this.lastAppliedLogIndexListeners.add(listener); + } + + private boolean enqueueTask(final EventTranslator tpl) { + if (this.shutdownLatch != null) { + // Shutting down + LOG.warn("FSMCaller is stopped, can not apply new task."); + return false; + } + this.taskQueue.publishEvent(tpl); + return true; + } + + @Override + public boolean onCommitted(final long committedIndex) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.COMMITTED; + task.committedIndex = committedIndex; + }); + } + + /** + * Flush all events in disruptor. + */ + @OnlyForTest + void flush() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + enqueueTask((task, sequence) -> { + task.type = TaskType.FLUSH; + task.shutdownLatch = latch; + }); + latch.await(); + } + + @Override + public boolean onSnapshotLoad(final LoadSnapshotClosure done) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.SNAPSHOT_LOAD; + task.done = done; + }); + } + + @Override + public boolean onSnapshotSave(final SaveSnapshotClosure done) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.SNAPSHOT_SAVE; + task.done = done; + }); + } + + @Override + public boolean onLeaderStop(final Status status) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.LEADER_STOP; + task.status = new Status(status); + }); + } + + @Override + public boolean onLeaderStart(final long term) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.LEADER_START; + task.term = term; + }); + } + + @Override + public boolean onStartFollowing(final LeaderChangeContext ctx) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.START_FOLLOWING; + task.leaderChangeCtx = new LeaderChangeContext(ctx.getLeaderId(), ctx.getTerm(), ctx.getStatus()); + }); + } + + @Override + public boolean onStopFollowing(final LeaderChangeContext ctx) { + return enqueueTask((task, sequence) -> { + task.type = TaskType.STOP_FOLLOWING; + task.leaderChangeCtx = new LeaderChangeContext(ctx.getLeaderId(), ctx.getTerm(), ctx.getStatus()); + }); + } + + /** + * Closure runs with an error. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:20:31 PM + */ + public class OnErrorClosure implements Closure { + private RaftException error; + + public OnErrorClosure(final RaftException error) { + super(); + this.error = error; + } + + public RaftException getError() { + return this.error; + } + + public void setError(final RaftException error) { + this.error = error; + } + + @Override + public void run(final Status st) { + } + } + + @Override + public boolean onError(final RaftException error) { + if (!this.error.getStatus().isOk()) { + LOG.warn("FSMCaller already in error status, ignore new error.", error); + return false; + } + final OnErrorClosure c = new OnErrorClosure(error); + return enqueueTask((task, sequence) -> { + task.type = TaskType.ERROR; + task.done = c; + }); + } + + @Override + public long getLastAppliedIndex() { + return this.lastAppliedIndex.get(); + } + + @Override + public synchronized void join() throws InterruptedException { + if (this.shutdownLatch != null) { + this.shutdownLatch.await(); + this.disruptor.shutdown(); + if (this.afterShutdown != null) { + this.afterShutdown.run(Status.OK()); + this.afterShutdown = null; + } + this.shutdownLatch = null; + } + } + + @SuppressWarnings("ConstantConditions") + private long runApplyTask(final ApplyTask task, long maxCommittedIndex, final boolean endOfBatch) { + CountDownLatch shutdown = null; + if (task.type == TaskType.COMMITTED) { + if (task.committedIndex > maxCommittedIndex) { + maxCommittedIndex = task.committedIndex; + } + task.reset(); + } else { + if (maxCommittedIndex >= 0) { + this.currTask = TaskType.COMMITTED; + doCommitted(maxCommittedIndex); + maxCommittedIndex = -1L; // reset maxCommittedIndex + } + final long startMs = Utils.monotonicMs(); + try { + switch (task.type) { + case COMMITTED: + Requires.requireTrue(false, "Impossible"); + break; + case SNAPSHOT_SAVE: + this.currTask = TaskType.SNAPSHOT_SAVE; + if (passByStatus(task.done)) { + doSnapshotSave((SaveSnapshotClosure) task.done); + } + break; + case SNAPSHOT_LOAD: + this.currTask = TaskType.SNAPSHOT_LOAD; + if (passByStatus(task.done)) { + doSnapshotLoad((LoadSnapshotClosure) task.done); + } + break; + case LEADER_STOP: + this.currTask = TaskType.LEADER_STOP; + doLeaderStop(task.status); + break; + case LEADER_START: + this.currTask = TaskType.LEADER_START; + doLeaderStart(task.term); + break; + case START_FOLLOWING: + this.currTask = TaskType.START_FOLLOWING; + doStartFollowing(task.leaderChangeCtx); + break; + case STOP_FOLLOWING: + this.currTask = TaskType.STOP_FOLLOWING; + doStopFollowing(task.leaderChangeCtx); + break; + case ERROR: + this.currTask = TaskType.ERROR; + doOnError((OnErrorClosure) task.done); + break; + case IDLE: + Requires.requireTrue(false, "Can't reach here"); + break; + case SHUTDOWN: + this.currTask = TaskType.SHUTDOWN; + shutdown = task.shutdownLatch; + break; + case FLUSH: + this.currTask = TaskType.FLUSH; + shutdown = task.shutdownLatch; + break; + } + } finally { + this.nodeMetrics.recordLatency(task.type.metricName(), Utils.monotonicMs() - startMs); + task.reset(); + } + } + try { + if (endOfBatch && maxCommittedIndex >= 0) { + this.currTask = TaskType.COMMITTED; + doCommitted(maxCommittedIndex); + maxCommittedIndex = -1L; // reset maxCommittedIndex + } + this.currTask = TaskType.IDLE; + return maxCommittedIndex; + } finally { + if (shutdown != null) { + shutdown.countDown(); + } + } + } + + private void doShutdown() { + if (this.node != null) { + this.node = null; + } + if (this.fsm != null) { + this.fsm.onShutdown(); + } + } + + private void notifyLastAppliedIndexUpdated(final long lastAppliedIndex) { + for (final LastAppliedLogIndexListener listener : this.lastAppliedLogIndexListeners) { + listener.onApplied(lastAppliedIndex); + } + } + + private void doCommitted(final long committedIndex) { + if (!this.error.getStatus().isOk()) { + return; + } + final long lastAppliedIndex = this.lastAppliedIndex.get(); + // We can tolerate the disorder of committed_index + if (lastAppliedIndex >= committedIndex) { + return; + } + final long startMs = Utils.monotonicMs(); + try { + final List closures = new ArrayList<>(); + final List taskClosures = new ArrayList<>(); + final long firstClosureIndex = this.closureQueue.popClosureUntil(committedIndex, closures, taskClosures); + + // Calls TaskClosure#onCommitted if necessary + onTaskCommitted(taskClosures); + + Requires.requireTrue(firstClosureIndex >= 0, "Invalid firstClosureIndex"); + final IteratorImpl iterImpl = new IteratorImpl(this.fsm, this.logManager, closures, firstClosureIndex, + lastAppliedIndex, committedIndex, this.applyingIndex); + while (iterImpl.isGood()) { + final LogEntry logEntry = iterImpl.entry(); + if (logEntry.getType() != EnumOutter.EntryType.ENTRY_TYPE_DATA) { + if (logEntry.getType() == EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION) { + if (logEntry.getOldPeers() != null && !logEntry.getOldPeers().isEmpty()) { + // Joint stage is not supposed to be noticeable by end users. + this.fsm.onConfigurationCommitted(new Configuration(iterImpl.entry().getPeers())); + } + } + if (iterImpl.done() != null) { + // For other entries, we have nothing to do besides flush the + // pending tasks and run this closure to notify the caller that the + // entries before this one were successfully committed and applied. + iterImpl.done().run(Status.OK()); + } + iterImpl.next(); + continue; + } + + // Apply data task to user state machine + doApplyTasks(iterImpl); + } + + if (iterImpl.hasError()) { + setError(iterImpl.getError()); + iterImpl.runTheRestClosureWithError(); + } + final long lastIndex = iterImpl.getIndex() - 1; + final long lastTerm = this.logManager.getTerm(lastIndex); + final LogId lastAppliedId = new LogId(lastIndex, lastTerm); + this.lastAppliedIndex.set(lastIndex); + this.lastAppliedTerm = lastTerm; + this.logManager.setAppliedId(lastAppliedId); + notifyLastAppliedIndexUpdated(lastIndex); + } finally { + this.nodeMetrics.recordLatency("fsm-commit", Utils.monotonicMs() - startMs); + } + } + + private void onTaskCommitted(final List closures) { + for (int i = 0, size = closures.size(); i < size; i++) { + final TaskClosure done = closures.get(i); + done.onCommitted(); + } + } + + private void doApplyTasks(final IteratorImpl iterImpl) { + final IteratorWrapper iter = new IteratorWrapper(iterImpl); + final long startApplyMs = Utils.monotonicMs(); + final long startIndex = iter.getIndex(); + try { + this.fsm.onApply(iter); + } finally { + this.nodeMetrics.recordLatency("fsm-apply-tasks", Utils.monotonicMs() - startApplyMs); + this.nodeMetrics.recordSize("fsm-apply-tasks-count", iter.getIndex() - startIndex); + } + if (iter.hasNext()) { + LOG.error("Iterator is still valid, did you return before iterator reached the end?"); + } + // Try move to next in case that we pass the same log twice. + iter.next(); + } + + private void doSnapshotSave(final SaveSnapshotClosure done) { + Requires.requireNonNull(done, "SaveSnapshotClosure is null"); + final long lastAppliedIndex = this.lastAppliedIndex.get(); + final RaftOutter.SnapshotMeta.Builder metaBuilder = RaftOutter.SnapshotMeta.newBuilder() // + .setLastIncludedIndex(lastAppliedIndex) // + .setLastIncludedTerm(this.lastAppliedTerm); + final ConfigurationEntry confEntry = this.logManager.getConfiguration(lastAppliedIndex); + if (confEntry == null || confEntry.isEmpty()) { + LOG.error("Empty conf entry for lastAppliedIndex={}", lastAppliedIndex); + Utils.runClosureInThread(done, new Status(RaftError.EINVAL, "Empty conf entry for lastAppliedIndex=%s", + lastAppliedIndex)); + return; + } + for (final PeerId peer : confEntry.getConf()) { + metaBuilder.addPeers(peer.toString()); + } + for (final PeerId peer : confEntry.getConf().getLearners()) { + metaBuilder.addLearners(peer.toString()); + } + if (confEntry.getOldConf() != null) { + for (final PeerId peer : confEntry.getOldConf()) { + metaBuilder.addOldPeers(peer.toString()); + } + for (final PeerId peer : confEntry.getOldConf().getLearners()) { + metaBuilder.addOldLearners(peer.toString()); + } + } + final SnapshotWriter writer = done.start(metaBuilder.build()); + if (writer == null) { + done.run(new Status(RaftError.EINVAL, "snapshot_storage create SnapshotWriter failed")); + return; + } + this.fsm.onSnapshotSave(writer, done); + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("StateMachine ["); + switch (this.currTask) { + case IDLE: + sb.append("Idle"); + break; + case COMMITTED: + sb.append("Applying logIndex=").append(this.applyingIndex); + break; + case SNAPSHOT_SAVE: + sb.append("Saving snapshot"); + break; + case SNAPSHOT_LOAD: + sb.append("Loading snapshot"); + break; + case ERROR: + sb.append("Notifying error"); + break; + case LEADER_STOP: + sb.append("Notifying leader stop"); + break; + case LEADER_START: + sb.append("Notifying leader start"); + break; + case START_FOLLOWING: + sb.append("Notifying start following"); + break; + case STOP_FOLLOWING: + sb.append("Notifying stop following"); + break; + case SHUTDOWN: + sb.append("Shutting down"); + break; + default: + break; + } + return sb.append(']').toString(); + } + + private void doSnapshotLoad(final LoadSnapshotClosure done) { + Requires.requireNonNull(done, "LoadSnapshotClosure is null"); + final SnapshotReader reader = done.start(); + if (reader == null) { + done.run(new Status(RaftError.EINVAL, "open SnapshotReader failed")); + return; + } + final RaftOutter.SnapshotMeta meta = reader.load(); + if (meta == null) { + done.run(new Status(RaftError.EINVAL, "SnapshotReader load meta failed")); + if (reader.getRaftError() == RaftError.EIO) { + final RaftException err = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT, RaftError.EIO, + "Fail to load snapshot meta"); + setError(err); + } + return; + } + final LogId lastAppliedId = new LogId(this.lastAppliedIndex.get(), this.lastAppliedTerm); + final LogId snapshotId = new LogId(meta.getLastIncludedIndex(), meta.getLastIncludedTerm()); + if (lastAppliedId.compareTo(snapshotId) > 0) { + done.run(new Status( + RaftError.ESTALE, + "Loading a stale snapshot last_applied_index=%d last_applied_term=%d snapshot_index=%d snapshot_term=%d", + lastAppliedId.getIndex(), lastAppliedId.getTerm(), snapshotId.getIndex(), snapshotId.getTerm())); + return; + } + if (!this.fsm.onSnapshotLoad(reader)) { + done.run(new Status(-1, "StateMachine onSnapshotLoad failed")); + final RaftException e = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, + RaftError.ESTATEMACHINE, "StateMachine onSnapshotLoad failed"); + setError(e); + return; + } + if (meta.getOldPeersCount() == 0) { + // Joint stage is not supposed to be noticeable by end users. + final Configuration conf = new Configuration(); + for (int i = 0, size = meta.getPeersCount(); i < size; i++) { + final PeerId peer = new PeerId(); + Requires.requireTrue(peer.parse(meta.getPeers(i)), "Parse peer failed"); + conf.addPeer(peer); + } + this.fsm.onConfigurationCommitted(conf); + } + this.lastAppliedIndex.set(meta.getLastIncludedIndex()); + this.lastAppliedTerm = meta.getLastIncludedTerm(); + done.run(Status.OK()); + } + + private void doOnError(final OnErrorClosure done) { + setError(done.getError()); + } + + private void doLeaderStop(final Status status) { + this.fsm.onLeaderStop(status); + } + + private void doLeaderStart(final long term) { + this.fsm.onLeaderStart(term); + } + + private void doStartFollowing(final LeaderChangeContext ctx) { + this.fsm.onStartFollowing(ctx); + } + + private void doStopFollowing(final LeaderChangeContext ctx) { + this.fsm.onStopFollowing(ctx); + } + + private void setError(final RaftException e) { + if (this.error.getType() != EnumOutter.ErrorType.ERROR_TYPE_NONE) { + // already report + return; + } + this.error = e; + if (this.fsm != null) { + this.fsm.onError(e); + } + if (this.node != null) { + this.node.onError(e); + } + } + + @OnlyForTest + RaftException getError() { + return this.error; + } + + private boolean passByStatus(final Closure done) { + final Status status = this.error.getStatus(); + if (!status.isOk()) { + if (done != null) { + done.run(new Status(RaftError.EINVAL, "FSMCaller is in bad status=`%s`", status)); + return false; + } + } + return true; + } + + @Override + public void describe(final Printer out) { + out.print(" ") // + .println(toString()); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorImpl.java new file mode 100644 index 0000000..a272b2f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorImpl.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.error.LogEntryCorruptedException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.Utils; + +/** + * The iterator implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 3:28:37 PM + */ +public class IteratorImpl { + + private final StateMachine fsm; + private final LogManager logManager; + private final List closures; + private final long firstClosureIndex; + private long currentIndex; + private final long committedIndex; + private LogEntry currEntry = new LogEntry(); // blank entry + private final AtomicLong applyingIndex; + private RaftException error; + + public IteratorImpl(final StateMachine fsm, final LogManager logManager, final List closures, + final long firstClosureIndex, final long lastAppliedIndex, final long committedIndex, + final AtomicLong applyingIndex) { + super(); + this.fsm = fsm; + this.logManager = logManager; + this.closures = closures; + this.firstClosureIndex = firstClosureIndex; + this.currentIndex = lastAppliedIndex; + this.committedIndex = committedIndex; + this.applyingIndex = applyingIndex; + next(); + } + + @Override + public String toString() { + return "IteratorImpl [fsm=" + this.fsm + ", logManager=" + this.logManager + ", closures=" + this.closures + + ", firstClosureIndex=" + this.firstClosureIndex + ", currentIndex=" + this.currentIndex + + ", committedIndex=" + this.committedIndex + ", currEntry=" + this.currEntry + ", applyingIndex=" + + this.applyingIndex + ", error=" + this.error + "]"; + } + + public LogEntry entry() { + return this.currEntry; + } + + public RaftException getError() { + return this.error; + } + + public boolean isGood() { + return this.currentIndex <= this.committedIndex && !hasError(); + } + + public boolean hasError() { + return this.error != null; + } + + /** + * Move to next + */ + public void next() { + this.currEntry = null; //release current entry + //get next entry + if (this.currentIndex <= this.committedIndex) { + ++this.currentIndex; + if (this.currentIndex <= this.committedIndex) { + try { + this.currEntry = this.logManager.getEntry(this.currentIndex); + if (this.currEntry == null) { + getOrCreateError().setType(EnumOutter.ErrorType.ERROR_TYPE_LOG); + getOrCreateError().getStatus().setError(-1, + "Fail to get entry at index=%d while committed_index=%d", this.currentIndex, + this.committedIndex); + } + } catch (final LogEntryCorruptedException e) { + getOrCreateError().setType(EnumOutter.ErrorType.ERROR_TYPE_LOG); + getOrCreateError().getStatus().setError(RaftError.EINVAL, e.getMessage()); + } + this.applyingIndex.set(this.currentIndex); + } + } + } + + public long getIndex() { + return this.currentIndex; + } + + public Closure done() { + if (this.currentIndex < this.firstClosureIndex) { + return null; + } + return this.closures.get((int) (this.currentIndex - this.firstClosureIndex)); + } + + protected void runTheRestClosureWithError() { + for (long i = Math.max(this.currentIndex, this.firstClosureIndex); i <= this.committedIndex; i++) { + final Closure done = this.closures.get((int) (i - this.firstClosureIndex)); + if (done != null) { + Requires.requireNonNull(this.error, "error"); + Requires.requireNonNull(this.error.getStatus(), "error.status"); + final Status status = this.error.getStatus(); + Utils.runClosureInThread(done, status); + } + } + } + + public void setErrorAndRollback(final long ntail, final Status st) { + Requires.requireTrue(ntail > 0, "Invalid ntail=" + ntail); + if (this.currEntry == null || this.currEntry.getType() != EnumOutter.EntryType.ENTRY_TYPE_DATA) { + this.currentIndex -= ntail; + } else { + this.currentIndex -= ntail - 1; + } + this.currEntry = null; + getOrCreateError().setType(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE); + getOrCreateError().getStatus().setError(RaftError.ESTATEMACHINE, + "StateMachine meet critical error when applying one or more tasks since index=%d, %s", this.currentIndex, + st != null ? st.toString() : "none"); + + } + + private RaftException getOrCreateError() { + if (this.error == null) { + this.error = new RaftException(); + } + return this.error; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorWrapper.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorWrapper.java new file mode 100644 index 0000000..ce504d0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/IteratorWrapper.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.nio.ByteBuffer; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Iterator; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LogEntry; + +public class IteratorWrapper implements Iterator { + + private final IteratorImpl impl; + + public IteratorWrapper(IteratorImpl iterImpl) { + super(); + this.impl = iterImpl; + } + + @Override + public boolean hasNext() { + return this.impl.isGood() && this.impl.entry().getType() == EnumOutter.EntryType.ENTRY_TYPE_DATA; + } + + @Override + public ByteBuffer next() { + final ByteBuffer data = getData(); + if (hasNext()) { + this.impl.next(); + } + return data; + } + + @Override + public ByteBuffer getData() { + final LogEntry entry = this.impl.entry(); + return entry != null ? entry.getData() : null; + } + + @Override + public long getIndex() { + return this.impl.getIndex(); + } + + @Override + public long getTerm() { + return this.impl.entry().getId().getTerm(); + } + + @Override + public Closure done() { + return this.impl.done(); + } + + @Override + public void setErrorAndRollback(final long ntail, final Status st) { + this.impl.setErrorAndRollback(ntail, st); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeImpl.java new file mode 100644 index 0000000..b6a2653 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeImpl.java @@ -0,0 +1,3511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.stream.Collectors; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.JRaftServiceFactory; +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.ReadOnlyService; +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.CatchUpClosure; +import com.alipay.sofa.jraft.closure.ClosureQueue; +import com.alipay.sofa.jraft.closure.ClosureQueueImpl; +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.closure.SynchronizedClosure; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.conf.ConfigurationManager; +import com.alipay.sofa.jraft.entity.Ballot; +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.NodeId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.RaftOutter; +import com.alipay.sofa.jraft.entity.Task; +import com.alipay.sofa.jraft.entity.UserLog; +import com.alipay.sofa.jraft.error.LogIndexOutOfBoundsException; +import com.alipay.sofa.jraft.error.LogNotFoundException; +import com.alipay.sofa.jraft.error.OverloadException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.BallotBoxOptions; +import com.alipay.sofa.jraft.option.BootstrapOptions; +import com.alipay.sofa.jraft.option.FSMCallerOptions; +import com.alipay.sofa.jraft.option.LogManagerOptions; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.option.RaftMetaStorageOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.option.ReadOnlyOption; +import com.alipay.sofa.jraft.option.ReadOnlyServiceOptions; +import com.alipay.sofa.jraft.option.ReplicatorGroupOptions; +import com.alipay.sofa.jraft.option.SnapshotExecutorOptions; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter; +import com.alipay.sofa.jraft.rpc.impl.core.DefaultRaftClientService; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.LogStorage; +import com.alipay.sofa.jraft.storage.RaftMetaStorage; +import com.alipay.sofa.jraft.storage.SnapshotExecutor; +import com.alipay.sofa.jraft.storage.impl.LogManagerImpl; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotExecutorImpl; +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.DisruptorBuilder; +import com.alipay.sofa.jraft.util.DisruptorMetricSet; +import com.alipay.sofa.jraft.util.JRaftServiceLoader; +import com.alipay.sofa.jraft.util.JRaftSignalHandler; +import com.alipay.sofa.jraft.util.LogExceptionHandler; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Platform; +import com.alipay.sofa.jraft.util.RepeatedTimer; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.alipay.sofa.jraft.util.SignalHelper; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.alipay.sofa.jraft.util.ThreadId; +import com.alipay.sofa.jraft.util.Utils; +import com.alipay.sofa.jraft.util.concurrent.LongHeldDetectingReadWriteLock; +import com.alipay.sofa.jraft.util.timer.RaftTimerFactory; +import com.google.protobuf.Message; +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.EventTranslator; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +/** + * The raft replica node implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:26:51 PM + */ +public class NodeImpl implements Node, RaftServerService { + + private static final Logger LOG = LoggerFactory + .getLogger(NodeImpl.class); + + static { + try { + if (SignalHelper.supportSignal()) { + // TODO support windows signal + if (!Platform.isWindows()) { + final List handlers = JRaftServiceLoader.load(JRaftSignalHandler.class) // + .sort(); + SignalHelper.addSignal(SignalHelper.SIG_USR2, handlers); + } + } + } catch (final Throwable t) { + LOG.error("Fail to add signal.", t); + } + } + + public final static RaftTimerFactory TIMER_FACTORY = JRaftUtils + .raftTimerFactory(); + + public static final AtomicInteger GLOBAL_NUM_NODES = new AtomicInteger( + 0); + + /** Internal states */ + private final ReadWriteLock readWriteLock = new NodeReadWriteLock( + this); + protected final Lock writeLock = this.readWriteLock + .writeLock(); + protected final Lock readLock = this.readWriteLock + .readLock(); + private volatile State state; + private volatile CountDownLatch shutdownLatch; + private long currTerm; + private volatile long lastLeaderTimestamp; + private PeerId leaderId = new PeerId(); + private PeerId votedId; + private final Ballot voteCtx = new Ballot(); + private final Ballot prevVoteCtx = new Ballot(); + private ConfigurationEntry conf; + private StopTransferArg stopTransferArg; + /** Raft group and node options and identifier */ + private final String groupId; + private NodeOptions options; + private RaftOptions raftOptions; + private final PeerId serverId; + /** Other services */ + private final ConfigurationCtx confCtx; + private LogStorage logStorage; + private RaftMetaStorage metaStorage; + private ClosureQueue closureQueue; + private ConfigurationManager configManager; + private LogManager logManager; + private FSMCaller fsmCaller; + private BallotBox ballotBox; + private SnapshotExecutor snapshotExecutor; + private ReplicatorGroup replicatorGroup; + private final List shutdownContinuations = new ArrayList<>(); + private RaftClientService rpcService; + private ReadOnlyService readOnlyService; + /** Timers */ + private Scheduler timerManager; + private RepeatedTimer electionTimer; + private RepeatedTimer voteTimer; + private RepeatedTimer stepDownTimer; + private RepeatedTimer snapshotTimer; + private ScheduledFuture transferTimer; + private ThreadId wakingCandidate; + /** Disruptor to run node service */ + private Disruptor applyDisruptor; + private RingBuffer applyQueue; + + /** Metrics */ + private NodeMetrics metrics; + + private NodeId nodeId; + private JRaftServiceFactory serviceFactory; + + /** ReplicatorStateListeners */ + private final CopyOnWriteArrayList replicatorStateListeners = new CopyOnWriteArrayList<>(); + /** Node's target leader election priority value */ + private volatile int targetPriority; + /** The number of elections time out for current node */ + private volatile int electionTimeoutCounter; + + private static class NodeReadWriteLock extends LongHeldDetectingReadWriteLock { + + static final long MAX_BLOCKING_MS_TO_REPORT = SystemPropertyUtil.getLong( + "jraft.node.detecting.lock.max_blocking_ms_to_report", -1); + + private final Node node; + + public NodeReadWriteLock(final Node node) { + super(MAX_BLOCKING_MS_TO_REPORT, TimeUnit.MILLISECONDS); + this.node = node; + } + + @Override + public void report(final AcquireMode acquireMode, final Thread heldThread, + final Collection queuedThreads, final long blockedNanos) { + final long blockedMs = TimeUnit.NANOSECONDS.toMillis(blockedNanos); + LOG.warn( + "Raft-Node-Lock report: currentThread={}, acquireMode={}, heldThread={}, queuedThreads={}, blockedMs={}.", + Thread.currentThread(), acquireMode, heldThread, queuedThreads, blockedMs); + + final NodeMetrics metrics = this.node.getNodeMetrics(); + if (metrics != null) { + metrics.recordLatency("node-lock-blocked", blockedMs); + } + } + } + + /** + * Node service event. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:29:55 PM + */ + private static class LogEntryAndClosure { + LogEntry entry; + Closure done; + long expectedTerm; + CountDownLatch shutdownLatch; + + public void reset() { + this.entry = null; + this.done = null; + this.expectedTerm = 0; + this.shutdownLatch = null; + } + } + + private static class LogEntryAndClosureFactory implements EventFactory { + + @Override + public LogEntryAndClosure newInstance() { + return new LogEntryAndClosure(); + } + } + + /** + * Event handler. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:30:07 PM + */ + private class LogEntryAndClosureHandler implements EventHandler { + // task list for batch + private final List tasks = new ArrayList<>(NodeImpl.this.raftOptions.getApplyBatch()); + + @Override + public void onEvent(final LogEntryAndClosure event, final long sequence, final boolean endOfBatch) + throws Exception { + if (event.shutdownLatch != null) { + if (!this.tasks.isEmpty()) { + executeApplyingTasks(this.tasks); + reset(); + } + final int num = GLOBAL_NUM_NODES.decrementAndGet(); + LOG.info("The number of active nodes decrement to {}.", num); + event.shutdownLatch.countDown(); + return; + } + + this.tasks.add(event); + if (this.tasks.size() >= NodeImpl.this.raftOptions.getApplyBatch() || endOfBatch) { + executeApplyingTasks(this.tasks); + reset(); + } + } + + private void reset() { + for (final LogEntryAndClosure task : this.tasks) { + task.reset(); + } + this.tasks.clear(); + } + } + + /** + * Configuration commit context. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:29:38 PM + */ + private static class ConfigurationCtx { + enum Stage { + STAGE_NONE, // none stage + STAGE_CATCHING_UP, // the node is catching-up + STAGE_JOINT, // joint stage + STAGE_STABLE // stable stage + } + + final NodeImpl node; + Stage stage; + // Peers change times + int nchanges; + long version; + // peers + List newPeers = new ArrayList<>(); + List oldPeers = new ArrayList<>(); + List addingPeers = new ArrayList<>(); + // learners + List newLearners = new ArrayList<>(); + List oldLearners = new ArrayList<>(); + Closure done; + + public ConfigurationCtx(final NodeImpl node) { + super(); + this.node = node; + this.stage = Stage.STAGE_NONE; + this.version = 0; + this.done = null; + } + + /** + * Start change configuration. + */ + void start(final Configuration oldConf, final Configuration newConf, final Closure done) { + if (isBusy()) { + if (done != null) { + Utils.runClosureInThread(done, new Status(RaftError.EBUSY, "Already in busy stage.")); + } + throw new IllegalStateException("Busy stage"); + } + if (this.done != null) { + if (done != null) { + Utils.runClosureInThread(done, new Status(RaftError.EINVAL, "Already have done closure.")); + } + throw new IllegalArgumentException("Already have done closure"); + } + this.done = done; + this.stage = Stage.STAGE_CATCHING_UP; + this.oldPeers = oldConf.listPeers(); + this.newPeers = newConf.listPeers(); + this.oldLearners = oldConf.listLearners(); + this.newLearners = newConf.listLearners(); + final Configuration adding = new Configuration(); + final Configuration removing = new Configuration(); + newConf.diff(oldConf, adding, removing); + this.nchanges = adding.size() + removing.size(); + + addNewLearners(); + if (adding.isEmpty()) { + nextStage(); + return; + } + addNewPeers(adding); + } + + private void addNewPeers(final Configuration adding) { + this.addingPeers = adding.listPeers(); + LOG.info("Adding peers: {}.", this.addingPeers); + for (final PeerId newPeer : this.addingPeers) { + if (!this.node.replicatorGroup.addReplicator(newPeer)) { + LOG.error("Node {} start the replicator failed, peer={}.", this.node.getNodeId(), newPeer); + onCaughtUp(this.version, newPeer, false); + return; + } + final OnCaughtUp caughtUp = new OnCaughtUp(this.node, this.node.currTerm, newPeer, this.version); + final long dueTime = Utils.nowMs() + this.node.options.getElectionTimeoutMs(); + if (!this.node.replicatorGroup.waitCaughtUp(newPeer, this.node.options.getCatchupMargin(), dueTime, + caughtUp)) { + LOG.error("Node {} waitCaughtUp, peer={}.", this.node.getNodeId(), newPeer); + onCaughtUp(this.version, newPeer, false); + return; + } + } + } + + private void addNewLearners() { + final Set addingLearners = new HashSet<>(this.newLearners); + addingLearners.removeAll(this.oldLearners); + LOG.info("Adding learners: {}.", addingLearners); + for (final PeerId newLearner : addingLearners) { + if (!this.node.replicatorGroup.addReplicator(newLearner, ReplicatorType.Learner)) { + LOG.error("Node {} start the learner replicator failed, peer={}.", this.node.getNodeId(), + newLearner); + } + } + } + + void onCaughtUp(final long version, final PeerId peer, final boolean success) { + if (version != this.version) { + LOG.warn("Ignore onCaughtUp message, mismatch configuration context version, expect {}, but is {}.", + this.version, version); + return; + } + Requires.requireTrue(this.stage == Stage.STAGE_CATCHING_UP, "Stage is not in STAGE_CATCHING_UP"); + if (success) { + this.addingPeers.remove(peer); + if (this.addingPeers.isEmpty()) { + nextStage(); + return; + } + return; + } + LOG.warn("Node {} fail to catch up peer {} when trying to change peers from {} to {}.", + this.node.getNodeId(), peer, this.oldPeers, this.newPeers); + reset(new Status(RaftError.ECATCHUP, "Peer %s failed to catch up.", peer)); + } + + void reset() { + reset(null); + } + + void reset(final Status st) { + if (st != null && st.isOk()) { + this.node.stopReplicator(this.newPeers, this.oldPeers); + this.node.stopReplicator(this.newLearners, this.oldLearners); + } else { + this.node.stopReplicator(this.oldPeers, this.newPeers); + this.node.stopReplicator(this.oldLearners, this.newLearners); + } + clearPeers(); + clearLearners(); + + this.version++; + this.stage = Stage.STAGE_NONE; + this.nchanges = 0; + if (this.done != null) { + Utils.runClosureInThread(this.done, st != null ? st : new Status(RaftError.EPERM, + "Leader stepped down.")); + this.done = null; + } + } + + private void clearLearners() { + this.newLearners.clear(); + this.oldLearners.clear(); + } + + private void clearPeers() { + this.newPeers.clear(); + this.oldPeers.clear(); + this.addingPeers.clear(); + } + + /** + * Invoked when this node becomes the leader, write a configuration change log as the first log. + */ + void flush(final Configuration conf, final Configuration oldConf) { + Requires.requireTrue(!isBusy(), "Flush when busy"); + this.newPeers = conf.listPeers(); + this.newLearners = conf.listLearners(); + if (oldConf == null || oldConf.isEmpty()) { + this.stage = Stage.STAGE_STABLE; + this.oldPeers = this.newPeers; + this.oldLearners = this.newLearners; + } else { + this.stage = Stage.STAGE_JOINT; + this.oldPeers = oldConf.listPeers(); + this.oldLearners = oldConf.listLearners(); + } + this.node.unsafeApplyConfiguration(conf, oldConf == null || oldConf.isEmpty() ? null : oldConf, true); + } + + void nextStage() { + Requires.requireTrue(isBusy(), "Not in busy stage"); + switch (this.stage) { + case STAGE_CATCHING_UP: + if (this.nchanges > 0) { + this.stage = Stage.STAGE_JOINT; + this.node.unsafeApplyConfiguration(new Configuration(this.newPeers, this.newLearners), + new Configuration(this.oldPeers), false); + return; + } + case STAGE_JOINT: + this.stage = Stage.STAGE_STABLE; + this.node.unsafeApplyConfiguration(new Configuration(this.newPeers, this.newLearners), null, false); + break; + case STAGE_STABLE: + final boolean shouldStepDown = !this.newPeers.contains(this.node.serverId); + reset(new Status()); + if (shouldStepDown) { + this.node.stepDown(this.node.currTerm, true, new Status(RaftError.ELEADERREMOVED, + "This node was removed.")); + } + break; + case STAGE_NONE: + // noinspection ConstantConditions + Requires.requireTrue(false, "Can't reach here"); + break; + } + } + + boolean isBusy() { + return this.stage != Stage.STAGE_NONE; + } + } + + public NodeImpl() { + this(null, null); + } + + public NodeImpl(final String groupId, final PeerId serverId) { + super(); + if (groupId != null) { + Utils.verifyGroupId(groupId); + } + this.groupId = groupId; + this.serverId = serverId != null ? serverId.copy() : null; + this.state = State.STATE_UNINITIALIZED; + this.currTerm = 0; + updateLastLeaderTimestamp(Utils.monotonicMs()); + this.confCtx = new ConfigurationCtx(this); + this.wakingCandidate = null; + final int num = GLOBAL_NUM_NODES.incrementAndGet(); + LOG.info("The number of active nodes increment to {}.", num); + } + + private boolean initSnapshotStorage() { + if (StringUtils.isEmpty(this.options.getSnapshotUri())) { + LOG.warn("Do not set snapshot uri, ignore initSnapshotStorage."); + return true; + } + this.snapshotExecutor = new SnapshotExecutorImpl(); + final SnapshotExecutorOptions opts = new SnapshotExecutorOptions(); + opts.setUri(this.options.getSnapshotUri()); + opts.setFsmCaller(this.fsmCaller); + opts.setNode(this); + opts.setLogManager(this.logManager); + opts.setAddr(this.serverId != null ? this.serverId.getEndpoint() : null); + opts.setInitTerm(this.currTerm); + opts.setFilterBeforeCopyRemote(this.options.isFilterBeforeCopyRemote()); + // get snapshot throttle + opts.setSnapshotThrottle(this.options.getSnapshotThrottle()); + return this.snapshotExecutor.init(opts); + } + + private boolean initLogStorage() { + Requires.requireNonNull(this.fsmCaller, "Null fsm caller"); + this.logStorage = this.serviceFactory.createLogStorage(this.options.getLogUri(), this.raftOptions); + this.logManager = new LogManagerImpl(); + final LogManagerOptions opts = new LogManagerOptions(); + opts.setLogEntryCodecFactory(this.serviceFactory.createLogEntryCodecFactory()); + opts.setLogStorage(this.logStorage); + opts.setConfigurationManager(this.configManager); + opts.setFsmCaller(this.fsmCaller); + opts.setNodeMetrics(this.metrics); + opts.setDisruptorBufferSize(this.raftOptions.getDisruptorBufferSize()); + opts.setRaftOptions(this.raftOptions); + return this.logManager.init(opts); + } + + private boolean initMetaStorage() { + this.metaStorage = this.serviceFactory.createRaftMetaStorage(this.options.getRaftMetaUri(), this.raftOptions); + RaftMetaStorageOptions opts = new RaftMetaStorageOptions(); + opts.setNode(this); + if (!this.metaStorage.init(opts)) { + LOG.error("Node {} init meta storage failed, uri={}.", this.serverId, this.options.getRaftMetaUri()); + return false; + } + this.currTerm = this.metaStorage.getTerm(); + this.votedId = this.metaStorage.getVotedFor().copy(); + return true; + } + + private void handleSnapshotTimeout() { + this.writeLock.lock(); + try { + if (!this.state.isActive()) { + return; + } + } finally { + this.writeLock.unlock(); + } + // do_snapshot in another thread to avoid blocking the timer thread. + Utils.runInThread(() -> doSnapshot(null)); + } + + private void handleElectionTimeout() { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (this.state != State.STATE_FOLLOWER) { + return; + } + if (isCurrentLeaderValid()) { + return; + } + resetLeaderId(PeerId.emptyPeer(), new Status(RaftError.ERAFTTIMEDOUT, "Lost connection from leader %s.", + this.leaderId)); + + // Judge whether to launch a election. + if (!allowLaunchElection()) { + return; + } + + doUnlock = false; + preVote(); + + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + /** + * Whether to allow for launching election or not by comparing node's priority with target + * priority. And at the same time, if next leader is not elected until next election + * timeout, it decays its local target priority exponentially. + * + * @return Whether current node will launch election or not. + */ + @SuppressWarnings("NonAtomicOperationOnVolatileField") + private boolean allowLaunchElection() { + + // Priority 0 is a special value so that a node will never participate in election. + if (this.serverId.isPriorityNotElected()) { + LOG.warn("Node {} will never participate in election, because it's priority={}.", getNodeId(), + this.serverId.getPriority()); + return false; + } + + // If this nodes disable priority election, then it can make a election. + if (this.serverId.isPriorityDisabled()) { + return true; + } + + // If current node's priority < target_priority, it does not initiate leader, + // election and waits for the next election timeout. + if (this.serverId.getPriority() < this.targetPriority) { + this.electionTimeoutCounter++; + + // If next leader is not elected until next election timeout, it + // decays its local target priority exponentially. + if (this.electionTimeoutCounter > 1) { + decayTargetPriority(); + this.electionTimeoutCounter = 0; + } + + if (this.electionTimeoutCounter == 1) { + LOG.debug("Node {} does not initiate leader election and waits for the next election timeout.", + getNodeId()); + return false; + } + } + + return this.serverId.getPriority() >= this.targetPriority; + } + + /** + * Decay targetPriority value based on gap value. + */ + @SuppressWarnings("NonAtomicOperationOnVolatileField") + private void decayTargetPriority() { + // Default Gap value should be bigger than 10. + final int decayPriorityGap = Math.max(this.options.getDecayPriorityGap(), 10); + final int gap = Math.max(decayPriorityGap, (this.targetPriority / 5)); + + final int prevTargetPriority = this.targetPriority; + this.targetPriority = Math.max(ElectionPriority.MinValue, (this.targetPriority - gap)); + LOG.info("Node {} priority decay, from: {}, to: {}.", getNodeId(), prevTargetPriority, this.targetPriority); + } + + /** + * Check and set configuration for node.At the same time, if configuration is changed, + * then compute and update the target priority value. + * + * @param inLock whether the writeLock has already been locked in other place. + * + */ + private void checkAndSetConfiguration(final boolean inLock) { + if (!inLock) { + this.writeLock.lock(); + } + try { + final ConfigurationEntry prevConf = this.conf; + this.conf = this.logManager.checkAndSetConfiguration(prevConf); + + if (this.conf != prevConf) { + // Update target priority value + final int prevTargetPriority = this.targetPriority; + this.targetPriority = getMaxPriorityOfNodes(this.conf.getConf().getPeers()); + if (prevTargetPriority != this.targetPriority) { + LOG.info("Node {} target priority value has changed from: {}, to: {}.", getNodeId(), + prevTargetPriority, this.targetPriority); + } + this.electionTimeoutCounter = 0; + } + } finally { + if (!inLock) { + this.writeLock.unlock(); + } + } + } + + /** + * Get max priority value for all nodes in the same Raft group, and update current node's target priority value. + * + * @param peerIds peer nodes in the same Raft group + * + */ + private int getMaxPriorityOfNodes(final List peerIds) { + Requires.requireNonNull(peerIds, "Null peer list"); + + int maxPriority = Integer.MIN_VALUE; + for (final PeerId peerId : peerIds) { + final int priorityVal = peerId.getPriority(); + maxPriority = Math.max(priorityVal, maxPriority); + } + + return maxPriority; + } + + private boolean initFSMCaller(final LogId bootstrapId) { + if (this.fsmCaller == null) { + LOG.error("Fail to init fsm caller, null instance, bootstrapId={}.", bootstrapId); + return false; + } + this.closureQueue = new ClosureQueueImpl(); + final FSMCallerOptions opts = new FSMCallerOptions(); + opts.setAfterShutdown(status -> afterShutdown()); + opts.setLogManager(this.logManager); + opts.setFsm(this.options.getFsm()); + opts.setClosureQueue(this.closureQueue); + opts.setNode(this); + opts.setBootstrapId(bootstrapId); + opts.setDisruptorBufferSize(this.raftOptions.getDisruptorBufferSize()); + return this.fsmCaller.init(opts); + } + + private static class BootstrapStableClosure extends LogManager.StableClosure { + + private final SynchronizedClosure done = new SynchronizedClosure(); + + public BootstrapStableClosure() { + super(null); + } + + public Status await() throws InterruptedException { + return this.done.await(); + } + + @Override + public void run(final Status status) { + this.done.run(status); + } + } + + public boolean bootstrap(final BootstrapOptions opts) throws InterruptedException { + if (opts.getLastLogIndex() > 0 && (opts.getGroupConf().isEmpty() || opts.getFsm() == null)) { + LOG.error("Invalid arguments for bootstrap, groupConf={}, fsm={}, lastLogIndex={}.", opts.getGroupConf(), + opts.getFsm(), opts.getLastLogIndex()); + return false; + } + if (opts.getGroupConf().isEmpty()) { + LOG.error("Bootstrapping an empty node makes no sense."); + return false; + } + Requires.requireNonNull(opts.getServiceFactory(), "Null jraft service factory"); + this.serviceFactory = opts.getServiceFactory(); + // Term is not an option since changing it is very dangerous + final long bootstrapLogTerm = opts.getLastLogIndex() > 0 ? 1 : 0; + final LogId bootstrapId = new LogId(opts.getLastLogIndex(), bootstrapLogTerm); + this.options = new NodeOptions(); + this.raftOptions = this.options.getRaftOptions(); + this.metrics = new NodeMetrics(opts.isEnableMetrics()); + this.options.setFsm(opts.getFsm()); + this.options.setLogUri(opts.getLogUri()); + this.options.setRaftMetaUri(opts.getRaftMetaUri()); + this.options.setSnapshotUri(opts.getSnapshotUri()); + + this.configManager = new ConfigurationManager(); + // Create fsmCaller at first as logManager needs it to report error + this.fsmCaller = new FSMCallerImpl(); + + if (!initLogStorage()) { + LOG.error("Fail to init log storage."); + return false; + } + if (!initMetaStorage()) { + LOG.error("Fail to init meta storage."); + return false; + } + if (this.currTerm == 0) { + this.currTerm = 1; + if (!this.metaStorage.setTermAndVotedFor(1, new PeerId())) { + LOG.error("Fail to set term."); + return false; + } + } + + if (opts.getFsm() != null && !initFSMCaller(bootstrapId)) { + LOG.error("Fail to init fsm caller."); + return false; + } + + final LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION); + entry.getId().setTerm(this.currTerm); + entry.setPeers(opts.getGroupConf().listPeers()); + entry.setLearners(opts.getGroupConf().listLearners()); + + final List entries = new ArrayList<>(); + entries.add(entry); + + final BootstrapStableClosure bootstrapDone = new BootstrapStableClosure(); + this.logManager.appendEntries(entries, bootstrapDone); + if (!bootstrapDone.await().isOk()) { + LOG.error("Fail to append configuration."); + return false; + } + + if (opts.getLastLogIndex() > 0) { + if (!initSnapshotStorage()) { + LOG.error("Fail to init snapshot storage."); + return false; + } + final SynchronizedClosure snapshotDone = new SynchronizedClosure(); + this.snapshotExecutor.doSnapshot(snapshotDone); + if (!snapshotDone.await().isOk()) { + LOG.error("Fail to save snapshot, status={}.", snapshotDone.getStatus()); + return false; + } + } + + if (this.logManager.getFirstLogIndex() != opts.getLastLogIndex() + 1) { + throw new IllegalStateException("First and last log index mismatch"); + } + if (opts.getLastLogIndex() > 0) { + if (this.logManager.getLastLogIndex() != opts.getLastLogIndex()) { + throw new IllegalStateException("Last log index mismatch"); + } + } else { + if (this.logManager.getLastLogIndex() != opts.getLastLogIndex() + 1) { + throw new IllegalStateException("Last log index mismatch"); + } + } + + return true; + } + + private int heartbeatTimeout(final int electionTimeout) { + return Math.max(electionTimeout / this.raftOptions.getElectionHeartbeatFactor(), 10); + } + + private int randomTimeout(final int timeoutMs) { + return ThreadLocalRandom.current().nextInt(timeoutMs, timeoutMs + this.raftOptions.getMaxElectionDelayMs()); + } + + @Override + public boolean init(final NodeOptions opts) { + Requires.requireNonNull(opts, "Null node options"); + Requires.requireNonNull(opts.getRaftOptions(), "Null raft options"); + Requires.requireNonNull(opts.getServiceFactory(), "Null jraft service factory"); + this.serviceFactory = opts.getServiceFactory(); + this.options = opts; + this.raftOptions = opts.getRaftOptions(); + this.metrics = new NodeMetrics(opts.isEnableMetrics()); + this.serverId.setPriority(opts.getElectionPriority()); + this.electionTimeoutCounter = 0; + + if (this.serverId.getIp().equals(Utils.IP_ANY)) { + LOG.error("Node can't started from IP_ANY."); + return false; + } + + if (!NodeManager.getInstance().serverExists(this.serverId.getEndpoint())) { + LOG.error("No RPC server attached to, did you forget to call addService?"); + return false; + } + + this.timerManager = TIMER_FACTORY.getRaftScheduler(this.options.isSharedTimerPool(), + this.options.getTimerPoolSize(), "JRaft-Node-ScheduleThreadPool"); + + // Init timers + final String suffix = getNodeId().toString(); + String name = "JRaft-VoteTimer-" + suffix; + this.voteTimer = new RepeatedTimer(name, this.options.getElectionTimeoutMs(), TIMER_FACTORY.getVoteTimer( + this.options.isSharedVoteTimer(), name)) { + + @Override + protected void onTrigger() { + handleVoteTimeout(); + } + + @Override + protected int adjustTimeout(final int timeoutMs) { + return randomTimeout(timeoutMs); + } + }; + name = "JRaft-ElectionTimer-" + suffix; + this.electionTimer = new RepeatedTimer(name, this.options.getElectionTimeoutMs(), + TIMER_FACTORY.getElectionTimer(this.options.isSharedElectionTimer(), name)) { + + @Override + protected void onTrigger() { + handleElectionTimeout(); + } + + @Override + protected int adjustTimeout(final int timeoutMs) { + return randomTimeout(timeoutMs); + } + }; + name = "JRaft-StepDownTimer-" + suffix; + this.stepDownTimer = new RepeatedTimer(name, this.options.getElectionTimeoutMs() >> 1, + TIMER_FACTORY.getStepDownTimer(this.options.isSharedStepDownTimer(), name)) { + + @Override + protected void onTrigger() { + handleStepDownTimeout(); + } + }; + name = "JRaft-SnapshotTimer-" + suffix; + this.snapshotTimer = new RepeatedTimer(name, this.options.getSnapshotIntervalSecs() * 1000, + TIMER_FACTORY.getSnapshotTimer(this.options.isSharedSnapshotTimer(), name)) { + + private volatile boolean firstSchedule = true; + + @Override + protected void onTrigger() { + handleSnapshotTimeout(); + } + + @Override + protected int adjustTimeout(final int timeoutMs) { + if (!this.firstSchedule) { + return timeoutMs; + } + + // Randomize the first snapshot trigger timeout + this.firstSchedule = false; + if (timeoutMs > 0) { + int half = timeoutMs / 2; + return half + ThreadLocalRandom.current().nextInt(half); + } else { + return timeoutMs; + } + } + }; + + this.configManager = new ConfigurationManager(); + + this.applyDisruptor = DisruptorBuilder. newInstance() // + .setRingBufferSize(this.raftOptions.getDisruptorBufferSize()) // + .setEventFactory(new LogEntryAndClosureFactory()) // + .setThreadFactory(new NamedThreadFactory("JRaft-NodeImpl-Disruptor-", true)) // + .setProducerType(ProducerType.MULTI) // + .setWaitStrategy(new BlockingWaitStrategy()) // + .build(); + this.applyDisruptor.handleEventsWith(new LogEntryAndClosureHandler()); + this.applyDisruptor.setDefaultExceptionHandler(new LogExceptionHandler(getClass().getSimpleName())); + this.applyQueue = this.applyDisruptor.start(); + if (this.metrics.getMetricRegistry() != null) { + this.metrics.getMetricRegistry().register("jraft-node-impl-disruptor", + new DisruptorMetricSet(this.applyQueue)); + } + + this.fsmCaller = new FSMCallerImpl(); + if (!initLogStorage()) { + LOG.error("Node {} initLogStorage failed.", getNodeId()); + return false; + } + if (!initMetaStorage()) { + LOG.error("Node {} initMetaStorage failed.", getNodeId()); + return false; + } + if (!initFSMCaller(new LogId(0, 0))) { + LOG.error("Node {} initFSMCaller failed.", getNodeId()); + return false; + } + this.ballotBox = new BallotBox(); + final BallotBoxOptions ballotBoxOpts = new BallotBoxOptions(); + ballotBoxOpts.setWaiter(this.fsmCaller); + ballotBoxOpts.setClosureQueue(this.closureQueue); + if (!this.ballotBox.init(ballotBoxOpts)) { + LOG.error("Node {} init ballotBox failed.", getNodeId()); + return false; + } + + if (!initSnapshotStorage()) { + LOG.error("Node {} initSnapshotStorage failed.", getNodeId()); + return false; + } + + final Status st = this.logManager.checkConsistency(); + if (!st.isOk()) { + LOG.error("Node {} is initialized with inconsistent log, status={}.", getNodeId(), st); + return false; + } + this.conf = new ConfigurationEntry(); + this.conf.setId(new LogId()); + // if have log using conf in log, else using conf in options + if (this.logManager.getLastLogIndex() > 0) { + checkAndSetConfiguration(false); + } else { + this.conf.setConf(this.options.getInitialConf()); + // initially set to max(priority of all nodes) + this.targetPriority = getMaxPriorityOfNodes(this.conf.getConf().getPeers()); + } + + if (!this.conf.isEmpty()) { + Requires.requireTrue(this.conf.isValid(), "Invalid conf: %s", this.conf); + } else { + LOG.info("Init node {} with empty conf.", this.serverId); + } + + // TODO RPC service and ReplicatorGroup is in cycle dependent, refactor it + this.replicatorGroup = new ReplicatorGroupImpl(); + this.rpcService = new DefaultRaftClientService(this.replicatorGroup); + final ReplicatorGroupOptions rgOpts = new ReplicatorGroupOptions(); + rgOpts.setHeartbeatTimeoutMs(heartbeatTimeout(this.options.getElectionTimeoutMs())); + rgOpts.setElectionTimeoutMs(this.options.getElectionTimeoutMs()); + rgOpts.setLogManager(this.logManager); + rgOpts.setBallotBox(this.ballotBox); + rgOpts.setNode(this); + rgOpts.setRaftRpcClientService(this.rpcService); + rgOpts.setSnapshotStorage(this.snapshotExecutor != null ? this.snapshotExecutor.getSnapshotStorage() : null); + rgOpts.setRaftOptions(this.raftOptions); + rgOpts.setTimerManager(this.timerManager); + + // Adds metric registry to RPC service. + this.options.setMetricRegistry(this.metrics.getMetricRegistry()); + + if (!this.rpcService.init(this.options)) { + LOG.error("Fail to init rpc service."); + return false; + } + this.replicatorGroup.init(new NodeId(this.groupId, this.serverId), rgOpts); + + this.readOnlyService = new ReadOnlyServiceImpl(); + final ReadOnlyServiceOptions rosOpts = new ReadOnlyServiceOptions(); + rosOpts.setFsmCaller(this.fsmCaller); + rosOpts.setNode(this); + rosOpts.setRaftOptions(this.raftOptions); + + if (!this.readOnlyService.init(rosOpts)) { + LOG.error("Fail to init readOnlyService."); + return false; + } + + // set state to follower + this.state = State.STATE_FOLLOWER; + + if (LOG.isInfoEnabled()) { + LOG.info("Node {} init, term={}, lastLogId={}, conf={}, oldConf={}.", getNodeId(), this.currTerm, + this.logManager.getLastLogId(false), this.conf.getConf(), this.conf.getOldConf()); + } + + if (this.snapshotExecutor != null && this.options.getSnapshotIntervalSecs() > 0) { + LOG.debug("Node {} start snapshot timer, term={}.", getNodeId(), this.currTerm); + this.snapshotTimer.start(); + } + + if (!this.conf.isEmpty()) { + stepDown(this.currTerm, false, new Status()); + } + + if (!NodeManager.getInstance().add(this)) { + LOG.error("NodeManager add {} failed.", getNodeId()); + return false; + } + + // Now the raft node is started , have to acquire the writeLock to avoid race + // conditions + this.writeLock.lock(); + if (this.conf.isStable() && this.conf.getConf().size() == 1 && this.conf.getConf().contains(this.serverId)) { + // The group contains only this server which must be the LEADER, trigger + // the timer immediately. + electSelf(); + } else { + this.writeLock.unlock(); + } + + return true; + } + + @OnlyForTest + void tryElectSelf() { + this.writeLock.lock(); + // unlock in electSelf + electSelf(); + } + + // should be in writeLock + private void electSelf() { + long oldTerm; + try { + LOG.info("Node {} start vote and grant vote self, term={}.", getNodeId(), this.currTerm); + if (!this.conf.contains(this.serverId)) { + LOG.warn("Node {} can't do electSelf as it is not in {}.", getNodeId(), this.conf); + return; + } + if (this.state == State.STATE_FOLLOWER) { + LOG.debug("Node {} stop election timer, term={}.", getNodeId(), this.currTerm); + this.electionTimer.stop(); + } + resetLeaderId(PeerId.emptyPeer(), new Status(RaftError.ERAFTTIMEDOUT, + "A follower's leader_id is reset to NULL as it begins to request_vote.")); + this.state = State.STATE_CANDIDATE; + this.currTerm++; + this.votedId = this.serverId.copy(); + LOG.debug("Node {} start vote timer, term={} .", getNodeId(), this.currTerm); + this.voteTimer.start(); + this.voteCtx.init(this.conf.getConf(), this.conf.isStable() ? null : this.conf.getOldConf()); + oldTerm = this.currTerm; + } finally { + this.writeLock.unlock(); + } + + final LogId lastLogId = this.logManager.getLastLogId(true); + + this.writeLock.lock(); + try { + // vote need defense ABA after unlock&writeLock + if (oldTerm != this.currTerm) { + LOG.warn("Node {} raise term {} when getLastLogId.", getNodeId(), this.currTerm); + return; + } + for (final PeerId peer : this.conf.listPeers()) { + if (peer.equals(this.serverId)) { + continue; + } + if (!this.rpcService.connect(peer.getEndpoint())) { + LOG.warn("Node {} channel init failed, address={}.", getNodeId(), peer.getEndpoint()); + continue; + } + final OnRequestVoteRpcDone done = new OnRequestVoteRpcDone(peer, this.currTerm, this); + done.request = RequestVoteRequest.newBuilder() // + .setPreVote(false) // It's not a pre-vote request. + .setGroupId(this.groupId) // + .setServerId(this.serverId.toString()) // + .setPeerId(peer.toString()) // + .setTerm(this.currTerm) // + .setLastLogIndex(lastLogId.getIndex()) // + .setLastLogTerm(lastLogId.getTerm()) // + .build(); + this.rpcService.requestVote(peer.getEndpoint(), done.request, done); + } + + this.metaStorage.setTermAndVotedFor(this.currTerm, this.serverId); + this.voteCtx.grant(this.serverId); + if (this.voteCtx.isGranted()) { + becomeLeader(); + } + } finally { + this.writeLock.unlock(); + } + } + + private void resetLeaderId(final PeerId newLeaderId, final Status status) { + if (newLeaderId.isEmpty()) { + if (!this.leaderId.isEmpty() && this.state.compareTo(State.STATE_TRANSFERRING) > 0) { + this.fsmCaller.onStopFollowing(new LeaderChangeContext(this.leaderId.copy(), this.currTerm, status)); + } + this.leaderId = PeerId.emptyPeer(); + } else { + if (this.leaderId == null || this.leaderId.isEmpty()) { + this.fsmCaller.onStartFollowing(new LeaderChangeContext(newLeaderId, this.currTerm, status)); + } + this.leaderId = newLeaderId.copy(); + } + } + + // in writeLock + private void checkStepDown(final long requestTerm, final PeerId serverId) { + final Status status = new Status(); + if (requestTerm > this.currTerm) { + status.setError(RaftError.ENEWLEADER, "Raft node receives message from new leader with higher term."); + stepDown(requestTerm, false, status); + } else if (this.state != State.STATE_FOLLOWER) { + status.setError(RaftError.ENEWLEADER, "Candidate receives message from new leader with the same term."); + stepDown(requestTerm, false, status); + } else if (this.leaderId.isEmpty()) { + status.setError(RaftError.ENEWLEADER, "Follower receives message from new leader with the same term."); + stepDown(requestTerm, false, status); + } + // save current leader + if (this.leaderId == null || this.leaderId.isEmpty()) { + resetLeaderId(serverId, status); + } + } + + private void becomeLeader() { + Requires.requireTrue(this.state == State.STATE_CANDIDATE, "Illegal state: " + this.state); + LOG.info("Node {} become leader of group, term={}, conf={}, oldConf={}.", getNodeId(), this.currTerm, + this.conf.getConf(), this.conf.getOldConf()); + // cancel candidate vote timer + stopVoteTimer(); + this.state = State.STATE_LEADER; + this.leaderId = this.serverId.copy(); + this.replicatorGroup.resetTerm(this.currTerm); + // Start follower's replicators + for (final PeerId peer : this.conf.listPeers()) { + if (peer.equals(this.serverId)) { + continue; + } + LOG.debug("Node {} add a replicator, term={}, peer={}.", getNodeId(), this.currTerm, peer); + if (!this.replicatorGroup.addReplicator(peer)) { + LOG.error("Fail to add a replicator, peer={}.", peer); + } + } + + // Start learner's replicators + for (final PeerId peer : this.conf.listLearners()) { + LOG.debug("Node {} add a learner replicator, term={}, peer={}.", getNodeId(), this.currTerm, peer); + if (!this.replicatorGroup.addReplicator(peer, ReplicatorType.Learner)) { + LOG.error("Fail to add a learner replicator, peer={}.", peer); + } + } + + // init commit manager + this.ballotBox.resetPendingIndex(this.logManager.getLastLogIndex() + 1); + // Register _conf_ctx to reject configuration changing before the first log + // is committed. + if (this.confCtx.isBusy()) { + throw new IllegalStateException(); + } + this.confCtx.flush(this.conf.getConf(), this.conf.getOldConf()); + this.stepDownTimer.start(); + } + + // should be in writeLock + private void stepDown(final long term, final boolean wakeupCandidate, final Status status) { + LOG.debug("Node {} stepDown, term={}, newTerm={}, wakeupCandidate={}.", getNodeId(), this.currTerm, term, + wakeupCandidate); + if (!this.state.isActive()) { + return; + } + if (this.state == State.STATE_CANDIDATE) { + stopVoteTimer(); + } else if (this.state.compareTo(State.STATE_TRANSFERRING) <= 0) { + stopStepDownTimer(); + this.ballotBox.clearPendingTasks(); + // signal fsm leader stop immediately + if (this.state == State.STATE_LEADER) { + onLeaderStop(status); + } + } + // reset leader_id + resetLeaderId(PeerId.emptyPeer(), status); + + // soft state in memory + this.state = State.STATE_FOLLOWER; + this.confCtx.reset(); + updateLastLeaderTimestamp(Utils.monotonicMs()); + if (this.snapshotExecutor != null) { + this.snapshotExecutor.interruptDownloadingSnapshots(term); + } + + // meta state + if (term > this.currTerm) { + this.currTerm = term; + this.votedId = PeerId.emptyPeer(); + this.metaStorage.setTermAndVotedFor(term, this.votedId); + } + + if (wakeupCandidate) { + this.wakingCandidate = this.replicatorGroup.stopAllAndFindTheNextCandidate(this.conf); + if (this.wakingCandidate != null) { + Replicator.sendTimeoutNowAndStop(this.wakingCandidate, this.options.getElectionTimeoutMs()); + } + } else { + this.replicatorGroup.stopAll(); + } + if (this.stopTransferArg != null) { + if (this.transferTimer != null) { + this.transferTimer.cancel(true); + } + // There is at most one StopTransferTimer at the same term, it's safe to + // mark stopTransferArg to NULL + this.stopTransferArg = null; + } + // Learner node will not trigger the election timer. + if (!isLearner()) { + this.electionTimer.restart(); + } else { + LOG.info("Node {} is a learner, election timer is not started.", this.nodeId); + } + } + + // Should be in readLock + private boolean isLearner() { + return this.conf.listLearners().contains(this.serverId); + } + + private void stopStepDownTimer() { + if (this.stepDownTimer != null) { + this.stepDownTimer.stop(); + } + } + + private void stopVoteTimer() { + if (this.voteTimer != null) { + this.voteTimer.stop(); + } + } + + class LeaderStableClosure extends LogManager.StableClosure { + + public LeaderStableClosure(final List entries) { + super(entries); + } + + @Override + public void run(final Status status) { + if (status.isOk()) { + NodeImpl.this.ballotBox.commitAt(this.firstLogIndex, this.firstLogIndex + this.nEntries - 1, + NodeImpl.this.serverId); + } else { + LOG.error("Node {} append [{}, {}] failed, status={}.", getNodeId(), this.firstLogIndex, + this.firstLogIndex + this.nEntries - 1, status); + } + } + } + + private void executeApplyingTasks(final List tasks) { + this.writeLock.lock(); + try { + final int size = tasks.size(); + if (this.state != State.STATE_LEADER) { + final Status st = new Status(); + if (this.state != State.STATE_TRANSFERRING) { + st.setError(RaftError.EPERM, "Is not leader."); + } else { + st.setError(RaftError.EBUSY, "Is transferring leadership."); + } + LOG.debug("Node {} can't apply, status={}.", getNodeId(), st); + final List dones = tasks.stream().map(ele -> ele.done).collect(Collectors.toList()); + Utils.runInThread(() -> { + for (final Closure done : dones) { + done.run(st); + } + }); + return; + } + final List entries = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + final LogEntryAndClosure task = tasks.get(i); + if (task.expectedTerm != -1 && task.expectedTerm != this.currTerm) { + LOG.debug("Node {} can't apply task whose expectedTerm={} doesn't match currTerm={}.", getNodeId(), + task.expectedTerm, this.currTerm); + if (task.done != null) { + final Status st = new Status(RaftError.EPERM, "expected_term=%d doesn't match current_term=%d", + task.expectedTerm, this.currTerm); + Utils.runClosureInThread(task.done, st); + task.reset(); + } + continue; + } + if (!this.ballotBox.appendPendingTask(this.conf.getConf(), + this.conf.isStable() ? null : this.conf.getOldConf(), task.done)) { + Utils.runClosureInThread(task.done, new Status(RaftError.EINTERNAL, "Fail to append task.")); + task.reset(); + continue; + } + // set task entry info before adding to list. + task.entry.getId().setTerm(this.currTerm); + task.entry.setType(EnumOutter.EntryType.ENTRY_TYPE_DATA); + entries.add(task.entry); + task.reset(); + } + this.logManager.appendEntries(entries, new LeaderStableClosure(entries)); + // update conf.first + checkAndSetConfiguration(true); + } finally { + this.writeLock.unlock(); + } + } + + /** + * Returns the node metrics. + * + * @return returns metrics of current node. + */ + @Override + public NodeMetrics getNodeMetrics() { + return this.metrics; + } + + /** + * Returns the JRaft service factory for current node. + * @since 1.2.6 + * @return the service factory + */ + public JRaftServiceFactory getServiceFactory() { + return this.serviceFactory; + } + + @Override + public void readIndex(final byte[] requestContext, final ReadIndexClosure done) { + if (this.shutdownLatch != null) { + Utils.runClosureInThread(done, new Status(RaftError.ENODESHUTDOWN, "Node is shutting down.")); + throw new IllegalStateException("Node is shutting down"); + } + Requires.requireNonNull(done, "Null closure"); + this.readOnlyService.addRequest(requestContext, done); + } + + /** + * ReadIndex response closure + * @author dennis + */ + private class ReadIndexHeartbeatResponseClosure extends RpcResponseClosureAdapter { + final ReadIndexResponse.Builder respBuilder; + final RpcResponseClosure closure; + final int quorum; + final int failPeersThreshold; + int ackSuccess; + int ackFailures; + boolean isDone; + + public ReadIndexHeartbeatResponseClosure(final RpcResponseClosure closure, + final ReadIndexResponse.Builder rb, final int quorum, + final int peersCount) { + super(); + this.closure = closure; + this.respBuilder = rb; + this.quorum = quorum; + this.failPeersThreshold = peersCount % 2 == 0 ? (quorum - 1) : quorum; + this.ackSuccess = 0; + this.ackFailures = 0; + this.isDone = false; + } + + @Override + public synchronized void run(final Status status) { + if (this.isDone) { + return; + } + if (status.isOk() && getResponse().getSuccess()) { + this.ackSuccess++; + } else { + this.ackFailures++; + } + // Include leader self vote yes. + if (this.ackSuccess + 1 >= this.quorum) { + this.respBuilder.setSuccess(true); + this.closure.setResponse(this.respBuilder.build()); + this.closure.run(Status.OK()); + this.isDone = true; + } else if (this.ackFailures >= this.failPeersThreshold) { + this.respBuilder.setSuccess(false); + this.closure.setResponse(this.respBuilder.build()); + this.closure.run(Status.OK()); + this.isDone = true; + } + } + } + + /** + * Handle read index request. + */ + @Override + public void handleReadIndexRequest(final ReadIndexRequest request, final RpcResponseClosure done) { + final long startMs = Utils.monotonicMs(); + this.readLock.lock(); + try { + switch (this.state) { + case STATE_LEADER: + readLeader(request, ReadIndexResponse.newBuilder(), done); + break; + case STATE_FOLLOWER: + readFollower(request, done); + break; + case STATE_TRANSFERRING: + done.run(new Status(RaftError.EBUSY, "Is transferring leadership.")); + break; + default: + done.run(new Status(RaftError.EPERM, "Invalid state for readIndex: %s.", this.state)); + break; + } + } finally { + this.readLock.unlock(); + this.metrics.recordLatency("handle-read-index", Utils.monotonicMs() - startMs); + this.metrics.recordSize("handle-read-index-entries", request.getEntriesCount()); + } + } + + private int getQuorum() { + final Configuration c = this.conf.getConf(); + if (c.isEmpty()) { + return 0; + } + return c.getPeers().size() / 2 + 1; + } + + private void readFollower(final ReadIndexRequest request, final RpcResponseClosure closure) { + if (this.leaderId == null || this.leaderId.isEmpty()) { + closure.run(new Status(RaftError.EPERM, "No leader at term %d.", this.currTerm)); + return; + } + // send request to leader. + final ReadIndexRequest newRequest = ReadIndexRequest.newBuilder() // + .mergeFrom(request) // + .setPeerId(this.leaderId.toString()) // + .build(); + this.rpcService.readIndex(this.leaderId.getEndpoint(), newRequest, -1, closure); + } + + private void readLeader(final ReadIndexRequest request, final ReadIndexResponse.Builder respBuilder, + final RpcResponseClosure closure) { + final int quorum = getQuorum(); + if (quorum <= 1) { + // Only one peer, fast path. + respBuilder.setSuccess(true) // + .setIndex(this.ballotBox.getLastCommittedIndex()); + closure.setResponse(respBuilder.build()); + closure.run(Status.OK()); + return; + } + + final long lastCommittedIndex = this.ballotBox.getLastCommittedIndex(); + if (this.logManager.getTerm(lastCommittedIndex) != this.currTerm) { + // Reject read only request when this leader has not committed any log entry at its term + closure + .run(new Status( + RaftError.EAGAIN, + "ReadIndex request rejected because leader has not committed any log entry at its term, logIndex=%d, currTerm=%d.", + lastCommittedIndex, this.currTerm)); + return; + } + respBuilder.setIndex(lastCommittedIndex); + + if (request.getPeerId() != null) { + // request from follower or learner, check if the follower/learner is in current conf. + final PeerId peer = new PeerId(); + peer.parse(request.getServerId()); + if (!this.conf.contains(peer) && !this.conf.containsLearner(peer)) { + closure + .run(new Status(RaftError.EPERM, "Peer %s is not in current configuration: %s.", peer, this.conf)); + return; + } + } + + ReadOnlyOption readOnlyOpt = this.raftOptions.getReadOnlyOptions(); + if (readOnlyOpt == ReadOnlyOption.ReadOnlyLeaseBased && !isLeaderLeaseValid()) { + // If leader lease timeout, we must change option to ReadOnlySafe + readOnlyOpt = ReadOnlyOption.ReadOnlySafe; + } + + switch (readOnlyOpt) { + case ReadOnlySafe: + final List peers = this.conf.getConf().getPeers(); + Requires.requireTrue(peers != null && !peers.isEmpty(), "Empty peers"); + final ReadIndexHeartbeatResponseClosure heartbeatDone = new ReadIndexHeartbeatResponseClosure(closure, + respBuilder, quorum, peers.size()); + // Send heartbeat requests to followers + for (final PeerId peer : peers) { + if (peer.equals(this.serverId)) { + continue; + } + this.replicatorGroup.sendHeartbeat(peer, heartbeatDone); + } + break; + case ReadOnlyLeaseBased: + // Responses to followers and local node. + respBuilder.setSuccess(true); + closure.setResponse(respBuilder.build()); + closure.run(Status.OK()); + break; + } + } + + @Override + public void apply(final Task task) { + if (this.shutdownLatch != null) { + Utils.runClosureInThread(task.getDone(), new Status(RaftError.ENODESHUTDOWN, "Node is shutting down.")); + throw new IllegalStateException("Node is shutting down"); + } + Requires.requireNonNull(task, "Null task"); + + final LogEntry entry = new LogEntry(); + entry.setData(task.getData()); + + final EventTranslator translator = (event, sequence) -> { + event.reset(); + event.done = task.getDone(); + event.entry = entry; + event.expectedTerm = task.getExpectedTerm(); + }; + + switch(this.options.getApplyTaskMode()) { + case Blocking: + this.applyQueue.publishEvent(translator); + break; + case NonBlocking: + default: + if (!this.applyQueue.tryPublishEvent(translator)) { + String errorMsg = "Node is busy, has too many tasks, queue is full and bufferSize="+ this.applyQueue.getBufferSize(); + Utils.runClosureInThread(task.getDone(), + new Status(RaftError.EBUSY, errorMsg)); + LOG.warn("Node {} applyQueue is overload.", getNodeId()); + this.metrics.recordTimes("apply-task-overload-times", 1); + if(task.getDone() == null) { + throw new OverloadException(errorMsg); + } + } + break; + } + } + + @Override + public Message handlePreVoteRequest(final RequestVoteRequest request) { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (!this.state.isActive()) { + LOG.warn("Node {} is not in active state, currTerm={}.", getNodeId(), this.currTerm); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(RequestVoteResponse.getDefaultInstance(), RaftError.EINVAL, + "Node %s is not in active state, state %s.", getNodeId(), this.state.name()); + } + final PeerId candidateId = new PeerId(); + if (!candidateId.parse(request.getServerId())) { + LOG.warn("Node {} received PreVoteRequest from {} serverId bad format.", getNodeId(), + request.getServerId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(RequestVoteResponse.getDefaultInstance(), RaftError.EINVAL, + "Parse candidateId failed: %s.", request.getServerId()); + } + boolean granted = false; + // noinspection ConstantConditions + do { + if (!this.conf.contains(candidateId)) { + LOG.warn("Node {} ignore PreVoteRequest from {} as it is not in conf <{}>.", getNodeId(), + request.getServerId(), this.conf); + break; + } + if (this.leaderId != null && !this.leaderId.isEmpty() && isCurrentLeaderValid()) { + LOG.info( + "Node {} ignore PreVoteRequest from {}, term={}, currTerm={}, because the leader {}'s lease is still valid.", + getNodeId(), request.getServerId(), request.getTerm(), this.currTerm, this.leaderId); + break; + } + if (request.getTerm() < this.currTerm) { + LOG.info("Node {} ignore PreVoteRequest from {}, term={}, currTerm={}.", getNodeId(), + request.getServerId(), request.getTerm(), this.currTerm); + // A follower replicator may not be started when this node become leader, so we must check it. + checkReplicator(candidateId); + break; + } + // A follower replicator may not be started when this node become leader, so we must check it. + // check replicator state + checkReplicator(candidateId); + + doUnlock = false; + this.writeLock.unlock(); + + final LogId lastLogId = this.logManager.getLastLogId(true); + + doUnlock = true; + this.writeLock.lock(); + final LogId requestLastLogId = new LogId(request.getLastLogIndex(), request.getLastLogTerm()); + granted = requestLastLogId.compareTo(lastLogId) >= 0; + + LOG.info( + "Node {} received PreVoteRequest from {}, term={}, currTerm={}, granted={}, requestLastLogId={}, lastLogId={}.", + getNodeId(), request.getServerId(), request.getTerm(), this.currTerm, granted, requestLastLogId, + lastLogId); + } while (false); + + return RequestVoteResponse.newBuilder() // + .setTerm(this.currTerm) // + .setGranted(granted) // + .build(); + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + // in read_lock + private boolean isLeaderLeaseValid() { + final long monotonicNowMs = Utils.monotonicMs(); + if (checkLeaderLease(monotonicNowMs)) { + return true; + } + checkDeadNodes0(this.conf.getConf().getPeers(), monotonicNowMs, false, null); + return checkLeaderLease(monotonicNowMs); + } + + private boolean checkLeaderLease(final long monotonicNowMs) { + return monotonicNowMs - this.lastLeaderTimestamp < this.options.getLeaderLeaseTimeoutMs(); + } + + private boolean isCurrentLeaderValid() { + return Utils.monotonicMs() - this.lastLeaderTimestamp < this.options.getElectionTimeoutMs(); + } + + private void updateLastLeaderTimestamp(final long lastLeaderTimestamp) { + this.lastLeaderTimestamp = lastLeaderTimestamp; + } + + private void checkReplicator(final PeerId candidateId) { + if (this.state == State.STATE_LEADER) { + this.replicatorGroup.checkReplicator(candidateId, false); + } + } + + @Override + public Message handleRequestVoteRequest(final RequestVoteRequest request) { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (!this.state.isActive()) { + LOG.warn("Node {} is not in active state, currTerm={}.", getNodeId(), this.currTerm); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(RequestVoteResponse.getDefaultInstance(), RaftError.EINVAL, + "Node %s is not in active state, state %s.", getNodeId(), this.state.name()); + } + final PeerId candidateId = new PeerId(); + if (!candidateId.parse(request.getServerId())) { + LOG.warn("Node {} received RequestVoteRequest from {} serverId bad format.", getNodeId(), + request.getServerId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(RequestVoteResponse.getDefaultInstance(), RaftError.EINVAL, + "Parse candidateId failed: %s.", request.getServerId()); + } + + // noinspection ConstantConditions + do { + // check term + if (request.getTerm() >= this.currTerm) { + LOG.info("Node {} received RequestVoteRequest from {}, term={}, currTerm={}.", getNodeId(), + request.getServerId(), request.getTerm(), this.currTerm); + // increase current term, change state to follower + if (request.getTerm() > this.currTerm) { + stepDown(request.getTerm(), false, new Status(RaftError.EHIGHERTERMRESPONSE, + "Raft node receives higher term RequestVoteRequest.")); + } + } else { + // ignore older term + LOG.info("Node {} ignore RequestVoteRequest from {}, term={}, currTerm={}.", getNodeId(), + request.getServerId(), request.getTerm(), this.currTerm); + break; + } + doUnlock = false; + this.writeLock.unlock(); + + final LogId lastLogId = this.logManager.getLastLogId(true); + + doUnlock = true; + this.writeLock.lock(); + // vote need ABA check after unlock&writeLock + if (request.getTerm() != this.currTerm) { + LOG.warn("Node {} raise term {} when get lastLogId.", getNodeId(), this.currTerm); + break; + } + + final boolean logIsOk = new LogId(request.getLastLogIndex(), request.getLastLogTerm()) + .compareTo(lastLogId) >= 0; + + if (logIsOk && (this.votedId == null || this.votedId.isEmpty())) { + stepDown(request.getTerm(), false, new Status(RaftError.EVOTEFORCANDIDATE, + "Raft node votes for some candidate, step down to restart election_timer.")); + this.votedId = candidateId.copy(); + this.metaStorage.setVotedFor(candidateId); + } + } while (false); + + return RequestVoteResponse.newBuilder() // + .setTerm(this.currTerm) // + .setGranted(request.getTerm() == this.currTerm && candidateId.equals(this.votedId)) // + .build(); + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + private static class FollowerStableClosure extends LogManager.StableClosure { + + final long committedIndex; + final AppendEntriesResponse.Builder responseBuilder; + final NodeImpl node; + final RpcRequestClosure done; + final long term; + + public FollowerStableClosure(final AppendEntriesRequest request, + final AppendEntriesResponse.Builder responseBuilder, final NodeImpl node, + final RpcRequestClosure done, final long term) { + super(null); + this.committedIndex = Math.min( + // committed index is likely less than the lastLogIndex + request.getCommittedIndex(), + // The logs after the appended entries can not be trust, so we can't commit them even if their indexes are less than request's committed index. + request.getPrevLogIndex() + request.getEntriesCount()); + this.responseBuilder = responseBuilder; + this.node = node; + this.done = done; + this.term = term; + } + + @Override + public void run(final Status status) { + + if (!status.isOk()) { + this.done.run(status); + return; + } + + this.node.readLock.lock(); + try { + if (this.term != this.node.currTerm) { + // The change of term indicates that leader has been changed during + // appending entries, so we can't respond ok to the old leader + // because we are not sure if the appended logs would be truncated + // by the new leader: + // - If they won't be truncated and we respond failure to the old + // leader, the new leader would know that they are stored in this + // peer and they will be eventually committed when the new leader + // found that quorum of the cluster have stored. + // - If they will be truncated and we responded success to the old + // leader, the old leader would possibly regard those entries as + // committed (very likely in a 3-nodes cluster) and respond + // success to the clients, which would break the rule that + // committed entries would never be truncated. + // So we have to respond failure to the old leader and set the new + // term to make it stepped down if it didn't. + this.responseBuilder.setSuccess(false).setTerm(this.node.currTerm); + this.done.sendResponse(this.responseBuilder.build()); + return; + } + } finally { + // It's safe to release lock as we know everything is ok at this point. + this.node.readLock.unlock(); + } + + // Don't touch node any more. + this.responseBuilder.setSuccess(true).setTerm(this.term); + + // Ballot box is thread safe and tolerates disorder. + this.node.ballotBox.setLastCommittedIndex(this.committedIndex); + + this.done.sendResponse(this.responseBuilder.build()); + } + } + + @Override + public Message handleAppendEntriesRequest(final AppendEntriesRequest request, final RpcRequestClosure done) { + boolean doUnlock = true; + final long startMs = Utils.monotonicMs(); + this.writeLock.lock(); + final int entriesCount = request.getEntriesCount(); + boolean success = false; + try { + if (!this.state.isActive()) { + LOG.warn("Node {} is not in active state, currTerm={}.", getNodeId(), this.currTerm); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(AppendEntriesResponse.getDefaultInstance(), RaftError.EINVAL, + "Node %s is not in active state, state %s.", getNodeId(), this.state.name()); + } + + final PeerId serverId = new PeerId(); + if (!serverId.parse(request.getServerId())) { + LOG.warn("Node {} received AppendEntriesRequest from {} serverId bad format.", getNodeId(), + request.getServerId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(AppendEntriesResponse.getDefaultInstance(), RaftError.EINVAL, + "Parse serverId failed: %s.", request.getServerId()); + } + + // Check stale term + if (request.getTerm() < this.currTerm) { + LOG.warn("Node {} ignore stale AppendEntriesRequest from {}, term={}, currTerm={}.", getNodeId(), + request.getServerId(), request.getTerm(), this.currTerm); + return AppendEntriesResponse.newBuilder() // + .setSuccess(false) // + .setTerm(this.currTerm) // + .build(); + } + + // Check term and state to step down + checkStepDown(request.getTerm(), serverId); + if (!serverId.equals(this.leaderId)) { + LOG.error("Another peer {} declares that it is the leader at term {} which was occupied by leader {}.", + serverId, this.currTerm, this.leaderId); + // Increase the term by 1 and make both leaders step down to minimize the + // loss of split brain + stepDown(request.getTerm() + 1, false, new Status(RaftError.ELEADERCONFLICT, + "More than one leader in the same term.")); + return AppendEntriesResponse.newBuilder() // + .setSuccess(false) // + .setTerm(request.getTerm() + 1) // + .build(); + } + + updateLastLeaderTimestamp(Utils.monotonicMs()); + + if (entriesCount > 0 && this.snapshotExecutor != null && this.snapshotExecutor.isInstallingSnapshot()) { + LOG.warn("Node {} received AppendEntriesRequest while installing snapshot.", getNodeId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(AppendEntriesResponse.getDefaultInstance(), RaftError.EBUSY, + "Node %s:%s is installing snapshot.", this.groupId, this.serverId); + } + + final long prevLogIndex = request.getPrevLogIndex(); + final long prevLogTerm = request.getPrevLogTerm(); + final long localPrevLogTerm = this.logManager.getTerm(prevLogIndex); + if (localPrevLogTerm != prevLogTerm) { + final long lastLogIndex = this.logManager.getLastLogIndex(); + + LOG.warn( + "Node {} reject term_unmatched AppendEntriesRequest from {}, term={}, prevLogIndex={}, prevLogTerm={}, localPrevLogTerm={}, lastLogIndex={}, entriesSize={}.", + getNodeId(), request.getServerId(), request.getTerm(), prevLogIndex, prevLogTerm, localPrevLogTerm, + lastLogIndex, entriesCount); + + return AppendEntriesResponse.newBuilder() // + .setSuccess(false) // + .setTerm(this.currTerm) // + .setLastLogIndex(lastLogIndex) // + .build(); + } + + if (entriesCount == 0) { + // heartbeat or probe request + final AppendEntriesResponse.Builder respBuilder = AppendEntriesResponse.newBuilder() // + .setSuccess(true) // + .setTerm(this.currTerm) // + .setLastLogIndex(this.logManager.getLastLogIndex()); + doUnlock = false; + this.writeLock.unlock(); + // see the comments at FollowerStableClosure#run() + this.ballotBox.setLastCommittedIndex(Math.min(request.getCommittedIndex(), prevLogIndex)); + return respBuilder.build(); + } + + // fast checking if log manager is overloaded + if (!this.logManager.hasAvailableCapacityToAppendEntries(1)) { + LOG.warn("Node {} received AppendEntriesRequest but log manager is busy.", getNodeId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(AppendEntriesResponse.getDefaultInstance(), RaftError.EBUSY, + "Node %s:%s log manager is busy.", this.groupId, this.serverId); + } + + // Parse request + long index = prevLogIndex; + final List entries = new ArrayList<>(entriesCount); + ByteBuffer allData = null; + if (request.hasData()) { + allData = request.getData().asReadOnlyByteBuffer(); + } + + final List entriesList = request.getEntriesList(); + for (int i = 0; i < entriesCount; i++) { + index++; + final RaftOutter.EntryMeta entry = entriesList.get(i); + + final LogEntry logEntry = logEntryFromMeta(index, allData, entry); + + if (logEntry != null) { + // Validate checksum + if (this.raftOptions.isEnableLogEntryChecksum() && logEntry.isCorrupted()) { + long realChecksum = logEntry.checksum(); + LOG.error( + "Corrupted log entry received from leader, index={}, term={}, expectedChecksum={}, realChecksum={}", + logEntry.getId().getIndex(), logEntry.getId().getTerm(), logEntry.getChecksum(), + realChecksum); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(AppendEntriesResponse.getDefaultInstance(), RaftError.EINVAL, + "The log entry is corrupted, index=%d, term=%d, expectedChecksum=%d, realChecksum=%d", + logEntry.getId().getIndex(), logEntry.getId().getTerm(), logEntry.getChecksum(), + realChecksum); + } + entries.add(logEntry); + } + } + + final FollowerStableClosure closure = new FollowerStableClosure(request, AppendEntriesResponse.newBuilder() + .setTerm(this.currTerm), this, done, this.currTerm); + this.logManager.appendEntries(entries, closure); + // update configuration after _log_manager updated its memory status + checkAndSetConfiguration(true); + success = true; + return null; + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + final long processLatency = Utils.monotonicMs() - startMs; + if (entriesCount == 0) { + this.metrics.recordLatency("handle-heartbeat-requests", processLatency); + } else { + this.metrics.recordLatency("handle-append-entries", processLatency); + } + if (success) { + // Don't stats heartbeat requests. + this.metrics.recordSize("handle-append-entries-count", entriesCount); + } + } + } + + private LogEntry logEntryFromMeta(final long index, final ByteBuffer allData, final RaftOutter.EntryMeta entry) { + if (entry.getType() != EnumOutter.EntryType.ENTRY_TYPE_UNKNOWN) { + final LogEntry logEntry = new LogEntry(); + logEntry.setId(new LogId(index, entry.getTerm())); + logEntry.setType(entry.getType()); + if (entry.hasChecksum()) { + logEntry.setChecksum(entry.getChecksum()); // since 1.2.6 + } + final long dataLen = entry.getDataLen(); + if (dataLen > 0) { + final byte[] bs = new byte[(int) dataLen]; + assert allData != null; + allData.get(bs, 0, bs.length); + logEntry.setData(ByteBuffer.wrap(bs)); + } + + if (entry.getPeersCount() > 0) { + if (entry.getType() != EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION) { + throw new IllegalStateException( + "Invalid log entry that contains peers but is not ENTRY_TYPE_CONFIGURATION type: " + + entry.getType()); + } + + fillLogEntryPeers(entry, logEntry); + } else if (entry.getType() == EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION) { + throw new IllegalStateException( + "Invalid log entry that contains zero peers but is ENTRY_TYPE_CONFIGURATION type"); + } + return logEntry; + } + return null; + } + + private void fillLogEntryPeers(final RaftOutter.EntryMeta entry, final LogEntry logEntry) { + // TODO refactor + if (entry.getPeersCount() > 0) { + final List peers = new ArrayList<>(entry.getPeersCount()); + for (final String peerStr : entry.getPeersList()) { + final PeerId peer = new PeerId(); + peer.parse(peerStr); + peers.add(peer); + } + logEntry.setPeers(peers); + } + + if (entry.getOldPeersCount() > 0) { + final List oldPeers = new ArrayList<>(entry.getOldPeersCount()); + for (final String peerStr : entry.getOldPeersList()) { + final PeerId peer = new PeerId(); + peer.parse(peerStr); + oldPeers.add(peer); + } + logEntry.setOldPeers(oldPeers); + } + + if (entry.getLearnersCount() > 0) { + final List peers = new ArrayList<>(entry.getLearnersCount()); + for (final String peerStr : entry.getLearnersList()) { + final PeerId peer = new PeerId(); + peer.parse(peerStr); + peers.add(peer); + } + logEntry.setLearners(peers); + } + + if (entry.getOldLearnersCount() > 0) { + final List peers = new ArrayList<>(entry.getOldLearnersCount()); + for (final String peerStr : entry.getOldLearnersList()) { + final PeerId peer = new PeerId(); + peer.parse(peerStr); + peers.add(peer); + } + logEntry.setOldLearners(peers); + } + } + + // called when leader receive greater term in AppendEntriesResponse + void increaseTermTo(final long newTerm, final Status status) { + this.writeLock.lock(); + try { + if (newTerm < this.currTerm) { + return; + } + stepDown(newTerm, false, status); + } finally { + this.writeLock.unlock(); + } + } + + /** + * Peer catch up callback + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-11 2:10:02 PM + */ + private static class OnCaughtUp extends CatchUpClosure { + private final NodeImpl node; + private final long term; + private final PeerId peer; + private final long version; + + public OnCaughtUp(final NodeImpl node, final long term, final PeerId peer, final long version) { + super(); + this.node = node; + this.term = term; + this.peer = peer; + this.version = version; + } + + @Override + public void run(final Status status) { + this.node.onCaughtUp(this.peer, this.term, this.version, status); + } + } + + private void onCaughtUp(final PeerId peer, final long term, final long version, final Status st) { + this.writeLock.lock(); + try { + // check current_term and state to avoid ABA problem + if (term != this.currTerm && this.state != State.STATE_LEADER) { + // term has changed and nothing should be done, otherwise there will be + // an ABA problem. + return; + } + if (st.isOk()) { + // Caught up successfully + this.confCtx.onCaughtUp(version, peer, true); + return; + } + // Retry if this peer is still alive + if (st.getCode() == RaftError.ETIMEDOUT.getNumber() + && Utils.monotonicMs() - this.replicatorGroup.getLastRpcSendTimestamp(peer) <= this.options + .getElectionTimeoutMs()) { + LOG.debug("Node {} waits peer {} to catch up.", getNodeId(), peer); + final OnCaughtUp caughtUp = new OnCaughtUp(this, term, peer, version); + final long dueTime = Utils.nowMs() + this.options.getElectionTimeoutMs(); + if (this.replicatorGroup.waitCaughtUp(peer, this.options.getCatchupMargin(), dueTime, caughtUp)) { + return; + } + LOG.warn("Node {} waitCaughtUp failed, peer={}.", getNodeId(), peer); + } + LOG.warn("Node {} caughtUp failed, status={}, peer={}.", getNodeId(), st, peer); + this.confCtx.onCaughtUp(version, peer, false); + } finally { + this.writeLock.unlock(); + } + } + + private boolean checkDeadNodes(final Configuration conf, final long monotonicNowMs, + final boolean stepDownOnCheckFail) { + // Check learner replicators at first. + for (final PeerId peer : conf.getLearners()) { + checkReplicator(peer); + } + // Ensure quorum nodes alive. + final List peers = conf.listPeers(); + final Configuration deadNodes = new Configuration(); + if (checkDeadNodes0(peers, monotonicNowMs, true, deadNodes)) { + return true; + } + if (stepDownOnCheckFail) { + LOG.warn("Node {} steps down when alive nodes don't satisfy quorum, term={}, deadNodes={}, conf={}.", + getNodeId(), this.currTerm, deadNodes, conf); + final Status status = new Status(); + status.setError(RaftError.ERAFTTIMEDOUT, "Majority of the group dies: %d/%d", deadNodes.size(), + peers.size()); + stepDown(this.currTerm, false, status); + } + return false; + } + + private boolean checkDeadNodes0(final List peers, final long monotonicNowMs, final boolean checkReplicator, + final Configuration deadNodes) { + final int leaderLeaseTimeoutMs = this.options.getLeaderLeaseTimeoutMs(); + int aliveCount = 0; + long startLease = Long.MAX_VALUE; + for (final PeerId peer : peers) { + if (peer.equals(this.serverId)) { + aliveCount++; + continue; + } + if (checkReplicator) { + checkReplicator(peer); + } + final long lastRpcSendTimestamp = this.replicatorGroup.getLastRpcSendTimestamp(peer); + if (monotonicNowMs - lastRpcSendTimestamp <= leaderLeaseTimeoutMs) { + aliveCount++; + if (startLease > lastRpcSendTimestamp) { + startLease = lastRpcSendTimestamp; + } + continue; + } + if (deadNodes != null) { + deadNodes.addPeer(peer); + } + } + if (aliveCount >= peers.size() / 2 + 1) { + updateLastLeaderTimestamp(startLease); + return true; + } + return false; + } + + // in read_lock + private List getAliveNodes(final Collection peers, final long monotonicNowMs) { + final int leaderLeaseTimeoutMs = this.options.getLeaderLeaseTimeoutMs(); + final List alivePeers = new ArrayList<>(); + for (final PeerId peer : peers) { + if (peer.equals(this.serverId)) { + alivePeers.add(peer.copy()); + continue; + } + if (monotonicNowMs - this.replicatorGroup.getLastRpcSendTimestamp(peer) <= leaderLeaseTimeoutMs) { + alivePeers.add(peer.copy()); + } + } + return alivePeers; + } + + @SuppressWarnings({ "LoopStatementThatDoesntLoop", "ConstantConditions" }) + private void handleStepDownTimeout() { + do { + this.readLock.lock(); + try { + if (this.state.compareTo(State.STATE_TRANSFERRING) > 0) { + LOG.debug("Node {} stop step-down timer, term={}, state={}.", getNodeId(), this.currTerm, + this.state); + return; + } + final long monotonicNowMs = Utils.monotonicMs(); + if (!checkDeadNodes(this.conf.getConf(), monotonicNowMs, false)) { + break; + } + if (!this.conf.getOldConf().isEmpty()) { + if (!checkDeadNodes(this.conf.getOldConf(), monotonicNowMs, false)) { + break; + } + } + return; + } finally { + this.readLock.unlock(); + } + } while (false); + + this.writeLock.lock(); + try { + if (this.state.compareTo(State.STATE_TRANSFERRING) > 0) { + LOG.debug("Node {} stop step-down timer, term={}, state={}.", getNodeId(), this.currTerm, this.state); + return; + } + final long monotonicNowMs = Utils.monotonicMs(); + checkDeadNodes(this.conf.getConf(), monotonicNowMs, true); + if (!this.conf.getOldConf().isEmpty()) { + checkDeadNodes(this.conf.getOldConf(), monotonicNowMs, true); + } + } finally { + this.writeLock.unlock(); + } + } + + /** + * Configuration changed callback. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-11 2:53:43 PM + */ + private class ConfigurationChangeDone implements Closure { + private final long term; + private final boolean leaderStart; + + public ConfigurationChangeDone(final long term, final boolean leaderStart) { + super(); + this.term = term; + this.leaderStart = leaderStart; + } + + @Override + public void run(final Status status) { + if (status.isOk()) { + onConfigurationChangeDone(this.term); + if (this.leaderStart) { + getOptions().getFsm().onLeaderStart(this.term); + } + } else { + LOG.error("Fail to run ConfigurationChangeDone, status: {}.", status); + } + } + } + + private void unsafeApplyConfiguration(final Configuration newConf, final Configuration oldConf, + final boolean leaderStart) { + Requires.requireTrue(this.confCtx.isBusy(), "ConfigurationContext is not busy"); + final LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION); + entry.setId(new LogId(0, this.currTerm)); + entry.setPeers(newConf.listPeers()); + entry.setLearners(newConf.listLearners()); + if (oldConf != null) { + entry.setOldPeers(oldConf.listPeers()); + entry.setOldLearners(oldConf.listLearners()); + } + final ConfigurationChangeDone configurationChangeDone = new ConfigurationChangeDone(this.currTerm, leaderStart); + // Use the new_conf to deal the quorum of this very log + if (!this.ballotBox.appendPendingTask(newConf, oldConf, configurationChangeDone)) { + Utils.runClosureInThread(configurationChangeDone, new Status(RaftError.EINTERNAL, "Fail to append task.")); + return; + } + final List entries = new ArrayList<>(); + entries.add(entry); + this.logManager.appendEntries(entries, new LeaderStableClosure(entries)); + checkAndSetConfiguration(false); + } + + private void unsafeRegisterConfChange(final Configuration oldConf, final Configuration newConf, final Closure done) { + + Requires.requireTrue(newConf.isValid(), "Invalid new conf: %s", newConf); + // The new conf entry(will be stored in log manager) should be valid + Requires.requireTrue(new ConfigurationEntry(null, newConf, oldConf).isValid(), "Invalid conf entry: %s", + newConf); + + if (this.state != State.STATE_LEADER) { + LOG.warn("Node {} refused configuration changing as the state={}.", getNodeId(), this.state); + if (done != null) { + final Status status = new Status(); + if (this.state == State.STATE_TRANSFERRING) { + status.setError(RaftError.EBUSY, "Is transferring leadership."); + } else { + status.setError(RaftError.EPERM, "Not leader"); + } + Utils.runClosureInThread(done, status); + } + return; + } + // check concurrent conf change + if (this.confCtx.isBusy()) { + LOG.warn("Node {} refused configuration concurrent changing.", getNodeId()); + if (done != null) { + Utils.runClosureInThread(done, new Status(RaftError.EBUSY, "Doing another configuration change.")); + } + return; + } + // Return immediately when the new peers equals to current configuration + if (this.conf.getConf().equals(newConf)) { + Utils.runClosureInThread(done); + return; + } + this.confCtx.start(oldConf, newConf, done); + } + + private void afterShutdown() { + List savedDoneList = null; + this.writeLock.lock(); + try { + if (!this.shutdownContinuations.isEmpty()) { + savedDoneList = new ArrayList<>(this.shutdownContinuations); + } + if (this.logStorage != null) { + this.logStorage.shutdown(); + } + this.state = State.STATE_SHUTDOWN; + } finally { + this.writeLock.unlock(); + } + if (savedDoneList != null) { + for (final Closure closure : savedDoneList) { + Utils.runClosureInThread(closure); + } + } + } + + @Override + public NodeOptions getOptions() { + return this.options; + } + + public Scheduler getTimerManager() { + return this.timerManager; + } + + @Override + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + @OnlyForTest + long getCurrentTerm() { + this.readLock.lock(); + try { + return this.currTerm; + } finally { + this.readLock.unlock(); + } + } + + @OnlyForTest + ConfigurationEntry getConf() { + this.readLock.lock(); + try { + return this.conf; + } finally { + this.readLock.unlock(); + } + } + + @Override + public void shutdown() { + shutdown(null); + } + + public void onConfigurationChangeDone(final long term) { + this.writeLock.lock(); + try { + if (term != this.currTerm || this.state.compareTo(State.STATE_TRANSFERRING) > 0) { + LOG.warn("Node {} process onConfigurationChangeDone at term {} while state={}, currTerm={}.", + getNodeId(), term, this.state, this.currTerm); + return; + } + this.confCtx.nextStage(); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public PeerId getLeaderId() { + this.readLock.lock(); + try { + return this.leaderId.isEmpty() ? null : this.leaderId; + } finally { + this.readLock.unlock(); + } + } + + @Override + public String getGroupId() { + return this.groupId; + } + + public PeerId getServerId() { + return this.serverId; + } + + @Override + public NodeId getNodeId() { + if (this.nodeId == null) { + this.nodeId = new NodeId(this.groupId, this.serverId); + } + return this.nodeId; + } + + public RaftClientService getRpcService() { + return this.rpcService; + } + + public void onError(final RaftException error) { + LOG.warn("Node {} got error: {}.", getNodeId(), error); + if (this.fsmCaller != null) { + // onError of fsmCaller is guaranteed to be executed once. + this.fsmCaller.onError(error); + } + if (this.readOnlyService != null) { + this.readOnlyService.setError(error); + } + this.writeLock.lock(); + try { + // If it is leader, need to wake up a new one; + // If it is follower, also step down to call on_stop_following. + if (this.state.compareTo(State.STATE_FOLLOWER) <= 0) { + stepDown(this.currTerm, this.state == State.STATE_LEADER, new Status(RaftError.EBADNODE, + "Raft node(leader or candidate) is in error.")); + } + if (this.state.compareTo(State.STATE_ERROR) < 0) { + this.state = State.STATE_ERROR; + } + } finally { + this.writeLock.unlock(); + } + } + + public void handleRequestVoteResponse(final PeerId peerId, final long term, final RequestVoteResponse response) { + this.writeLock.lock(); + try { + if (this.state != State.STATE_CANDIDATE) { + LOG.warn("Node {} received invalid RequestVoteResponse from {}, state not in STATE_CANDIDATE but {}.", + getNodeId(), peerId, this.state); + return; + } + // check stale term + if (term != this.currTerm) { + LOG.warn("Node {} received stale RequestVoteResponse from {}, term={}, currTerm={}.", getNodeId(), + peerId, term, this.currTerm); + return; + } + // check response term + if (response.getTerm() > this.currTerm) { + LOG.warn("Node {} received invalid RequestVoteResponse from {}, term={}, expect={}.", getNodeId(), + peerId, response.getTerm(), this.currTerm); + stepDown(response.getTerm(), false, new Status(RaftError.EHIGHERTERMRESPONSE, + "Raft node receives higher term request_vote_response.")); + return; + } + // check granted quorum? + if (response.getGranted()) { + this.voteCtx.grant(peerId); + if (this.voteCtx.isGranted()) { + becomeLeader(); + } + } + } finally { + this.writeLock.unlock(); + } + } + + private class OnRequestVoteRpcDone extends RpcResponseClosureAdapter { + + final long startMs; + final PeerId peer; + final long term; + final NodeImpl node; + RequestVoteRequest request; + + public OnRequestVoteRpcDone(final PeerId peer, final long term, final NodeImpl node) { + super(); + this.startMs = Utils.monotonicMs(); + this.peer = peer; + this.term = term; + this.node = node; + } + + @Override + public void run(final Status status) { + NodeImpl.this.metrics.recordLatency("request-vote", Utils.monotonicMs() - this.startMs); + if (!status.isOk()) { + LOG.warn("Node {} RequestVote to {} error: {}.", this.node.getNodeId(), this.peer, status); + } else { + this.node.handleRequestVoteResponse(this.peer, this.term, getResponse()); + } + } + } + + public void handlePreVoteResponse(final PeerId peerId, final long term, final RequestVoteResponse response) { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (this.state != State.STATE_FOLLOWER) { + LOG.warn("Node {} received invalid PreVoteResponse from {}, state not in STATE_FOLLOWER but {}.", + getNodeId(), peerId, this.state); + return; + } + if (term != this.currTerm) { + LOG.warn("Node {} received invalid PreVoteResponse from {}, term={}, currTerm={}.", getNodeId(), + peerId, term, this.currTerm); + return; + } + if (response.getTerm() > this.currTerm) { + LOG.warn("Node {} received invalid PreVoteResponse from {}, term {}, expect={}.", getNodeId(), peerId, + response.getTerm(), this.currTerm); + stepDown(response.getTerm(), false, new Status(RaftError.EHIGHERTERMRESPONSE, + "Raft node receives higher term pre_vote_response.")); + return; + } + LOG.info("Node {} received PreVoteResponse from {}, term={}, granted={}.", getNodeId(), peerId, + response.getTerm(), response.getGranted()); + // check granted quorum? + if (response.getGranted()) { + this.prevVoteCtx.grant(peerId); + if (this.prevVoteCtx.isGranted()) { + doUnlock = false; + electSelf(); + } + } + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + private class OnPreVoteRpcDone extends RpcResponseClosureAdapter { + + final long startMs; + final PeerId peer; + final long term; + RequestVoteRequest request; + + public OnPreVoteRpcDone(final PeerId peer, final long term) { + super(); + this.startMs = Utils.monotonicMs(); + this.peer = peer; + this.term = term; + } + + @Override + public void run(final Status status) { + NodeImpl.this.metrics.recordLatency("pre-vote", Utils.monotonicMs() - this.startMs); + if (!status.isOk()) { + LOG.warn("Node {} PreVote to {} error: {}.", getNodeId(), this.peer, status); + } else { + handlePreVoteResponse(this.peer, this.term, getResponse()); + } + } + } + + // in writeLock + private void preVote() { + long oldTerm; + try { + LOG.info("Node {} term {} start preVote.", getNodeId(), this.currTerm); + if (this.snapshotExecutor != null && this.snapshotExecutor.isInstallingSnapshot()) { + LOG.warn( + "Node {} term {} doesn't do preVote when installing snapshot as the configuration may be out of date.", + getNodeId(), this.currTerm); + return; + } + if (!this.conf.contains(this.serverId)) { + LOG.warn("Node {} can't do preVote as it is not in conf <{}>.", getNodeId(), this.conf); + return; + } + oldTerm = this.currTerm; + } finally { + this.writeLock.unlock(); + } + + final LogId lastLogId = this.logManager.getLastLogId(true); + + boolean doUnlock = true; + this.writeLock.lock(); + try { + // pre_vote need defense ABA after unlock&writeLock + if (oldTerm != this.currTerm) { + LOG.warn("Node {} raise term {} when get lastLogId.", getNodeId(), this.currTerm); + return; + } + this.prevVoteCtx.init(this.conf.getConf(), this.conf.isStable() ? null : this.conf.getOldConf()); + for (final PeerId peer : this.conf.listPeers()) { + if (peer.equals(this.serverId)) { + continue; + } + if (!this.rpcService.connect(peer.getEndpoint())) { + LOG.warn("Node {} channel init failed, address={}.", getNodeId(), peer.getEndpoint()); + continue; + } + final OnPreVoteRpcDone done = new OnPreVoteRpcDone(peer, this.currTerm); + done.request = RequestVoteRequest.newBuilder() // + .setPreVote(true) // it's a pre-vote request. + .setGroupId(this.groupId) // + .setServerId(this.serverId.toString()) // + .setPeerId(peer.toString()) // + .setTerm(this.currTerm + 1) // next term + .setLastLogIndex(lastLogId.getIndex()) // + .setLastLogTerm(lastLogId.getTerm()) // + .build(); + this.rpcService.preVote(peer.getEndpoint(), done.request, done); + } + this.prevVoteCtx.grant(this.serverId); + if (this.prevVoteCtx.isGranted()) { + doUnlock = false; + electSelf(); + } + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + private void handleVoteTimeout() { + this.writeLock.lock(); + if (this.state != State.STATE_CANDIDATE) { + this.writeLock.unlock(); + return; + } + + if (this.raftOptions.isStepDownWhenVoteTimedout()) { + LOG.warn( + "Candidate node {} term {} steps down when election reaching vote timeout: fail to get quorum vote-granted.", + this.nodeId, this.currTerm); + stepDown(this.currTerm, false, new Status(RaftError.ETIMEDOUT, + "Vote timeout: fail to get quorum vote-granted.")); + // unlock in preVote + preVote(); + } else { + LOG.debug("Node {} term {} retry to vote self.", getNodeId(), this.currTerm); + // unlock in electSelf + electSelf(); + } + } + + @Override + public boolean isLeader() { + return isLeader(true); + } + + @Override + public boolean isLeader(final boolean blocking) { + if (!blocking) { + return this.state == State.STATE_LEADER; + } + this.readLock.lock(); + try { + return this.state == State.STATE_LEADER; + } finally { + this.readLock.unlock(); + } + } + + @Override + public void shutdown(Closure done) { + List timers = null; + this.writeLock.lock(); + try { + LOG.info("Node {} shutdown, currTerm={} state={}.", getNodeId(), this.currTerm, this.state); + if (this.state.compareTo(State.STATE_SHUTTING) < 0) { + NodeManager.getInstance().remove(this); + // If it is leader, set the wakeup_a_candidate with true; + // If it is follower, call on_stop_following in step_down + if (this.state.compareTo(State.STATE_FOLLOWER) <= 0) { + stepDown(this.currTerm, this.state == State.STATE_LEADER, + new Status(RaftError.ESHUTDOWN, "Raft node is going to quit.")); + } + this.state = State.STATE_SHUTTING; + // Stop all timers + timers = stopAllTimers(); + if (this.readOnlyService != null) { + this.readOnlyService.shutdown(); + } + if (this.logManager != null) { + this.logManager.shutdown(); + } + if (this.metaStorage != null) { + this.metaStorage.shutdown(); + } + if (this.snapshotExecutor != null) { + this.snapshotExecutor.shutdown(); + } + if (this.wakingCandidate != null) { + Replicator.stop(this.wakingCandidate); + } + if (this.fsmCaller != null) { + this.fsmCaller.shutdown(); + } + if (this.rpcService != null) { + this.rpcService.shutdown(); + } + if (this.applyQueue != null) { + final CountDownLatch latch = new CountDownLatch(1); + this.shutdownLatch = latch; + Utils.runInThread( + () -> this.applyQueue.publishEvent((event, sequence) -> event.shutdownLatch = latch)); + } else { + final int num = GLOBAL_NUM_NODES.decrementAndGet(); + LOG.info("The number of active nodes decrement to {}.", num); + } + if (this.timerManager != null) { + this.timerManager.shutdown(); + } + } + + if (this.state != State.STATE_SHUTDOWN) { + if (done != null) { + this.shutdownContinuations.add(done); + done = null; + } + return; + } + } finally { + this.writeLock.unlock(); + + // Destroy all timers out of lock + if (timers != null) { + destroyAllTimers(timers); + } + // Call join() asynchronously + final Closure shutdownHook = done; + Utils.runInThread(() -> { + try { + join(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + // This node is down, it's ok to invoke done right now. Don't invoke this + // in place to avoid the dead writeLock issue when done.Run() is going to acquire + // a writeLock which is already held by the caller + if (shutdownHook != null) { + shutdownHook.run(Status.OK()); + } + } + }); + + } + } + + // Should in lock + private List stopAllTimers() { + final List timers = new ArrayList<>(); + if (this.electionTimer != null) { + this.electionTimer.stop(); + timers.add(this.electionTimer); + } + if (this.voteTimer != null) { + this.voteTimer.stop(); + timers.add(this.voteTimer); + } + if (this.stepDownTimer != null) { + this.stepDownTimer.stop(); + timers.add(this.stepDownTimer); + } + if (this.snapshotTimer != null) { + this.snapshotTimer.stop(); + timers.add(this.snapshotTimer); + } + return timers; + } + + private void destroyAllTimers(final List timers) { + for (final RepeatedTimer timer : timers) { + timer.destroy(); + } + } + + @Override + public synchronized void join() throws InterruptedException { + if (this.shutdownLatch != null) { + if (this.readOnlyService != null) { + this.readOnlyService.join(); + } + if (this.logManager != null) { + this.logManager.join(); + } + if (this.snapshotExecutor != null) { + this.snapshotExecutor.join(); + } + if (this.wakingCandidate != null) { + Replicator.join(this.wakingCandidate); + } + this.shutdownLatch.await(); + this.applyDisruptor.shutdown(); + this.applyQueue = null; + this.applyDisruptor = null; + this.shutdownLatch = null; + } + if (this.fsmCaller != null) { + this.fsmCaller.join(); + } + } + + private static class StopTransferArg { + final NodeImpl node; + final long term; + final PeerId peer; + + public StopTransferArg(final NodeImpl node, final long term, final PeerId peer) { + super(); + this.node = node; + this.term = term; + this.peer = peer; + } + } + + private void handleTransferTimeout(final long term, final PeerId peer) { + LOG.info("Node {} failed to transfer leadership to peer {}, reached timeout.", getNodeId(), peer); + this.writeLock.lock(); + try { + if (term == this.currTerm) { + this.replicatorGroup.stopTransferLeadership(peer); + if (this.state == State.STATE_TRANSFERRING) { + this.fsmCaller.onLeaderStart(term); + this.state = State.STATE_LEADER; + this.stopTransferArg = null; + } + } + } finally { + this.writeLock.unlock(); + } + } + + private void onTransferTimeout(final StopTransferArg arg) { + arg.node.handleTransferTimeout(arg.term, arg.peer); + } + + /** + * Retrieve current configuration this node seen so far. It's not a reliable way to + * retrieve cluster peers info, you should use {@link #listPeers()} instead. + * + * @return current configuration. + * + * @since 1.0.3 + */ + public Configuration getCurrentConf() { + this.readLock.lock(); + try { + if (this.conf != null && this.conf.getConf() != null) { + return this.conf.getConf().copy(); + } + return null; + } finally { + this.readLock.unlock(); + } + } + + @Override + public List listPeers() { + this.readLock.lock(); + try { + if (this.state != State.STATE_LEADER) { + throw new IllegalStateException("Not leader"); + } + return this.conf.getConf().listPeers(); + } finally { + this.readLock.unlock(); + } + } + + @Override + public List listAlivePeers() { + this.readLock.lock(); + try { + if (this.state != State.STATE_LEADER) { + throw new IllegalStateException("Not leader"); + } + return getAliveNodes(this.conf.getConf().getPeers(), Utils.monotonicMs()); + } finally { + this.readLock.unlock(); + } + } + + @Override + public List listLearners() { + this.readLock.lock(); + try { + if (this.state != State.STATE_LEADER) { + throw new IllegalStateException("Not leader"); + } + return this.conf.getConf().listLearners(); + } finally { + this.readLock.unlock(); + } + } + + @Override + public List listAliveLearners() { + this.readLock.lock(); + try { + if (this.state != State.STATE_LEADER) { + throw new IllegalStateException("Not leader"); + } + return getAliveNodes(this.conf.getConf().getLearners(), Utils.monotonicMs()); + } finally { + this.readLock.unlock(); + } + } + + @Override + public void addPeer(final PeerId peer, final Closure done) { + Requires.requireNonNull(peer, "Null peer"); + this.writeLock.lock(); + try { + Requires.requireTrue(!this.conf.getConf().contains(peer), "Peer already exists in current configuration"); + + final Configuration newConf = new Configuration(this.conf.getConf()); + newConf.addPeer(peer); + unsafeRegisterConfChange(this.conf.getConf(), newConf, done); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void removePeer(final PeerId peer, final Closure done) { + Requires.requireNonNull(peer, "Null peer"); + this.writeLock.lock(); + try { + Requires.requireTrue(this.conf.getConf().contains(peer), "Peer not found in current configuration"); + + final Configuration newConf = new Configuration(this.conf.getConf()); + newConf.removePeer(peer); + unsafeRegisterConfChange(this.conf.getConf(), newConf, done); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void changePeers(final Configuration newPeers, final Closure done) { + Requires.requireNonNull(newPeers, "Null new peers"); + Requires.requireTrue(!newPeers.isEmpty(), "Empty new peers"); + this.writeLock.lock(); + try { + LOG.info("Node {} change peers from {} to {}.", getNodeId(), this.conf.getConf(), newPeers); + unsafeRegisterConfChange(this.conf.getConf(), newPeers, done); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public Status resetPeers(final Configuration newPeers) { + Requires.requireNonNull(newPeers, "Null new peers"); + Requires.requireTrue(!newPeers.isEmpty(), "Empty new peers"); + Requires.requireTrue(newPeers.isValid(), "Invalid new peers: %s", newPeers); + this.writeLock.lock(); + try { + if (newPeers.isEmpty()) { + LOG.warn("Node {} set empty peers.", getNodeId()); + return new Status(RaftError.EINVAL, "newPeers is empty"); + } + if (!this.state.isActive()) { + LOG.warn("Node {} is in state {}, can't set peers.", getNodeId(), this.state); + return new Status(RaftError.EPERM, "Bad state: %s", this.state); + } + // bootstrap? + if (this.conf.getConf().isEmpty()) { + LOG.info("Node {} set peers to {} from empty.", getNodeId(), newPeers); + this.conf.setConf(newPeers); + stepDown(this.currTerm + 1, false, new Status(RaftError.ESETPEER, "Set peer from empty configuration")); + return Status.OK(); + } + if (this.state == State.STATE_LEADER && this.confCtx.isBusy()) { + LOG.warn("Node {} set peers need wait current conf changing.", getNodeId()); + return new Status(RaftError.EBUSY, "Changing to another configuration"); + } + // check equal, maybe retry direct return + if (this.conf.getConf().equals(newPeers)) { + return Status.OK(); + } + final Configuration newConf = new Configuration(newPeers); + LOG.info("Node {} set peers from {} to {}.", getNodeId(), this.conf.getConf(), newPeers); + this.conf.setConf(newConf); + this.conf.getOldConf().reset(); + stepDown(this.currTerm + 1, false, new Status(RaftError.ESETPEER, "Raft node set peer normally")); + return Status.OK(); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void addLearners(final List learners, final Closure done) { + checkPeers(learners); + this.writeLock.lock(); + try { + final Configuration newConf = new Configuration(this.conf.getConf()); + for (final PeerId peer : learners) { + newConf.addLearner(peer); + } + unsafeRegisterConfChange(this.conf.getConf(), newConf, done); + } finally { + this.writeLock.unlock(); + } + + } + + private void checkPeers(final List peers) { + Requires.requireNonNull(peers, "Null peers"); + Requires.requireTrue(!peers.isEmpty(), "Empty peers"); + for (final PeerId peer : peers) { + Requires.requireNonNull(peer, "Null peer"); + } + } + + @Override + public void removeLearners(final List learners, final Closure done) { + checkPeers(learners); + this.writeLock.lock(); + try { + final Configuration newConf = new Configuration(this.conf.getConf()); + for (final PeerId peer : learners) { + newConf.removeLearner(peer); + } + unsafeRegisterConfChange(this.conf.getConf(), newConf, done); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void resetLearners(final List learners, final Closure done) { + checkPeers(learners); + this.writeLock.lock(); + try { + final Configuration newConf = new Configuration(this.conf.getConf()); + newConf.setLearners(new LinkedHashSet<>(learners)); + unsafeRegisterConfChange(this.conf.getConf(), newConf, done); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void snapshot(final Closure done) { + doSnapshot(done); + } + + private void doSnapshot(final Closure done) { + if (this.snapshotExecutor != null) { + this.snapshotExecutor.doSnapshot(done); + } else { + if (done != null) { + final Status status = new Status(RaftError.EINVAL, "Snapshot is not supported"); + Utils.runClosureInThread(done, status); + } + } + } + + @Override + public void resetElectionTimeoutMs(final int electionTimeoutMs) { + Requires.requireTrue(electionTimeoutMs > 0, "Invalid electionTimeoutMs"); + this.writeLock.lock(); + try { + this.options.setElectionTimeoutMs(electionTimeoutMs); + this.replicatorGroup.resetHeartbeatInterval(heartbeatTimeout(this.options.getElectionTimeoutMs())); + this.replicatorGroup.resetElectionTimeoutInterval(electionTimeoutMs); + LOG.info("Node {} reset electionTimeout, currTimer {} state {} new electionTimeout {}.", getNodeId(), + this.currTerm, this.state, electionTimeoutMs); + this.electionTimer.reset(electionTimeoutMs); + } finally { + this.writeLock.unlock(); + } + } + + @Override + public Status transferLeadershipTo(final PeerId peer) { + Requires.requireNonNull(peer, "Null peer"); + this.writeLock.lock(); + try { + if (this.state != State.STATE_LEADER) { + LOG.warn("Node {} can't transfer leadership to peer {} as it is in state {}.", getNodeId(), peer, + this.state); + return new Status(this.state == State.STATE_TRANSFERRING ? RaftError.EBUSY : RaftError.EPERM, + "Not a leader"); + } + if (this.confCtx.isBusy()) { + // It's very messy to deal with the case when the |peer| received + // TimeoutNowRequest and increase the term while somehow another leader + // which was not replicated with the newest configuration has been + // elected. If no add_peer with this very |peer| is to be invoked ever + // after nor this peer is to be killed, this peer will spin in the voting + // procedure and make the each new leader stepped down when the peer + // reached vote timeout and it starts to vote (because it will increase + // the term of the group) + // To make things simple, refuse the operation and force users to + // invoke transfer_leadership_to after configuration changing is + // completed so that the peer's configuration is up-to-date when it + // receives the TimeOutNowRequest. + LOG.warn( + "Node {} refused to transfer leadership to peer {} when the leader is changing the configuration.", + getNodeId(), peer); + return new Status(RaftError.EBUSY, "Changing the configuration"); + } + + PeerId peerId = peer.copy(); + // if peer_id is ANY_PEER(0.0.0.0:0:0), the peer with the largest + // last_log_id will be selected. + if (peerId.equals(PeerId.ANY_PEER)) { + LOG.info("Node {} starts to transfer leadership to any peer.", getNodeId()); + if ((peerId = this.replicatorGroup.findTheNextCandidate(this.conf)) == null) { + return new Status(-1, "Candidate not found for any peer"); + } + } + if (peerId.equals(this.serverId)) { + LOG.info("Node {} transferred leadership to self.", this.serverId); + return Status.OK(); + } + if (!this.conf.contains(peerId)) { + LOG.info("Node {} refused to transfer leadership to peer {} as it is not in {}.", getNodeId(), peer, + this.conf); + return new Status(RaftError.EINVAL, "Not in current configuration"); + } + + final long lastLogIndex = this.logManager.getLastLogIndex(); + if (!this.replicatorGroup.transferLeadershipTo(peerId, lastLogIndex)) { + LOG.warn("No such peer {}.", peer); + return new Status(RaftError.EINVAL, "No such peer %s", peer); + } + this.state = State.STATE_TRANSFERRING; + final Status status = new Status(RaftError.ETRANSFERLEADERSHIP, + "Raft leader is transferring leadership to %s", peerId); + onLeaderStop(status); + LOG.info("Node {} starts to transfer leadership to peer {}.", getNodeId(), peer); + final StopTransferArg stopArg = new StopTransferArg(this, this.currTerm, peerId); + this.stopTransferArg = stopArg; + this.transferTimer = this.timerManager.schedule(() -> onTransferTimeout(stopArg), + this.options.getElectionTimeoutMs(), TimeUnit.MILLISECONDS); + + } finally { + this.writeLock.unlock(); + } + return Status.OK(); + } + + private void onLeaderStop(final Status status) { + this.replicatorGroup.clearFailureReplicators(); + this.fsmCaller.onLeaderStop(status); + } + + @Override + public Message handleTimeoutNowRequest(final TimeoutNowRequest request, final RpcRequestClosure done) { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (request.getTerm() != this.currTerm) { + final long savedCurrTerm = this.currTerm; + if (request.getTerm() > this.currTerm) { + stepDown(request.getTerm(), false, new Status(RaftError.EHIGHERTERMREQUEST, + "Raft node receives higher term request")); + } + LOG.info("Node {} received TimeoutNowRequest from {} while currTerm={} didn't match requestTerm={}.", + getNodeId(), request.getPeerId(), savedCurrTerm, request.getTerm()); + return TimeoutNowResponse.newBuilder() // + .setTerm(this.currTerm) // + .setSuccess(false) // + .build(); + } + if (this.state != State.STATE_FOLLOWER) { + LOG.info("Node {} received TimeoutNowRequest from {}, while state={}, term={}.", getNodeId(), + request.getServerId(), this.state, this.currTerm); + return TimeoutNowResponse.newBuilder() // + .setTerm(this.currTerm) // + .setSuccess(false) // + .build(); + } + + final long savedTerm = this.currTerm; + final TimeoutNowResponse resp = TimeoutNowResponse.newBuilder() // + .setTerm(this.currTerm + 1) // + .setSuccess(true) // + .build(); + // Parallelize response and election + done.sendResponse(resp); + doUnlock = false; + electSelf(); + LOG.info("Node {} received TimeoutNowRequest from {}, term={}.", getNodeId(), request.getServerId(), + savedTerm); + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + return null; + } + + @Override + public Message handleInstallSnapshot(final InstallSnapshotRequest request, final RpcRequestClosure done) { + if (this.snapshotExecutor == null) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINVAL, "Not supported snapshot"); + } + final PeerId serverId = new PeerId(); + if (!serverId.parse(request.getServerId())) { + LOG.warn("Node {} ignore InstallSnapshotRequest from {} bad server id.", getNodeId(), request.getServerId()); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINVAL, + "Parse serverId failed: %s", request.getServerId()); + } + + this.writeLock.lock(); + try { + if (!this.state.isActive()) { + LOG.warn("Node {} ignore InstallSnapshotRequest as it is not in active state {}.", getNodeId(), + this.state); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINVAL, + "Node %s:%s is not in active state, state %s.", this.groupId, this.serverId, this.state.name()); + } + + if (request.getTerm() < this.currTerm) { + LOG.warn("Node {} ignore stale InstallSnapshotRequest from {}, term={}, currTerm={}.", getNodeId(), + request.getPeerId(), request.getTerm(), this.currTerm); + return InstallSnapshotResponse.newBuilder() // + .setTerm(this.currTerm) // + .setSuccess(false) // + .build(); + } + + checkStepDown(request.getTerm(), serverId); + + if (!serverId.equals(this.leaderId)) { + LOG.error("Another peer {} declares that it is the leader at term {} which was occupied by leader {}.", + serverId, this.currTerm, this.leaderId); + // Increase the term by 1 and make both leaders step down to minimize the + // loss of split brain + stepDown(request.getTerm() + 1, false, new Status(RaftError.ELEADERCONFLICT, + "More than one leader in the same term.")); + return InstallSnapshotResponse.newBuilder() // + .setTerm(request.getTerm() + 1) // + .setSuccess(false) // + .build(); + } + + } finally { + this.writeLock.unlock(); + } + final long startMs = Utils.monotonicMs(); + try { + if (LOG.isInfoEnabled()) { + LOG.info( + "Node {} received InstallSnapshotRequest from {}, lastIncludedLogIndex={}, lastIncludedLogTerm={}, lastLogId={}.", + getNodeId(), request.getServerId(), request.getMeta().getLastIncludedIndex(), request.getMeta() + .getLastIncludedTerm(), this.logManager.getLastLogId(false)); + } + this.snapshotExecutor.installSnapshot(request, InstallSnapshotResponse.newBuilder(), done); + return null; + } finally { + this.metrics.recordLatency("install-snapshot", Utils.monotonicMs() - startMs); + } + } + + public void updateConfigurationAfterInstallingSnapshot() { + checkAndSetConfiguration(false); + } + + private void stopReplicator(final Collection keep, final Collection drop) { + if (drop != null) { + for (final PeerId peer : drop) { + if (!keep.contains(peer) && !peer.equals(this.serverId)) { + this.replicatorGroup.stopReplicator(peer); + } + } + } + } + + @Override + public UserLog readCommittedUserLog(final long index) { + if (index <= 0) { + throw new LogIndexOutOfBoundsException("Request index is invalid: " + index); + } + + final long savedLastAppliedIndex = this.fsmCaller.getLastAppliedIndex(); + + if (index > savedLastAppliedIndex) { + throw new LogIndexOutOfBoundsException("Request index " + index + " is greater than lastAppliedIndex: " + + savedLastAppliedIndex); + } + + long curIndex = index; + LogEntry entry = this.logManager.getEntry(curIndex); + if (entry == null) { + throw new LogNotFoundException("User log is deleted at index: " + index); + } + + do { + if (entry.getType() == EnumOutter.EntryType.ENTRY_TYPE_DATA) { + return new UserLog(curIndex, entry.getData()); + } else { + curIndex++; + } + if (curIndex > savedLastAppliedIndex) { + throw new IllegalStateException("No user log between index:" + index + " and last_applied_index:" + + savedLastAppliedIndex); + } + entry = this.logManager.getEntry(curIndex); + } while (entry != null); + + throw new LogNotFoundException("User log is deleted at index: " + curIndex); + } + + @Override + public void addReplicatorStateListener(final Replicator.ReplicatorStateListener replicatorStateListener) { + Requires.requireNonNull(replicatorStateListener, "replicatorStateListener"); + this.replicatorStateListeners.add(replicatorStateListener); + } + + @Override + public void removeReplicatorStateListener(final Replicator.ReplicatorStateListener replicatorStateListener) { + Requires.requireNonNull(replicatorStateListener, "replicatorStateListener"); + this.replicatorStateListeners.remove(replicatorStateListener); + } + + @Override + public void clearReplicatorStateListeners() { + this.replicatorStateListeners.clear(); + } + + @Override + public List getReplicatorStatueListeners() { + return this.replicatorStateListeners; + } + + @Override + public int getNodeTargetPriority() { + return this.targetPriority; + } + + @Override + public State getNodeState() { + return this.state; + } + + @Override + public void describe(final Printer out) { + // node + final String _nodeId; + final String _state; + final String _leaderId; + final long _currTerm; + final String _conf; + final int _targetPriority; + this.readLock.lock(); + try { + _nodeId = String.valueOf(getNodeId()); + _state = String.valueOf(this.state); + _leaderId = String.valueOf(this.leaderId); + _currTerm = this.currTerm; + _conf = String.valueOf(this.conf); + _targetPriority = this.targetPriority; + } finally { + this.readLock.unlock(); + } + out.print("nodeId: ") // + .println(_nodeId); + out.print("state: ") // + .println(_state); + out.print("leaderId: ") // + .println(_leaderId); + out.print("term: ") // + .println(_currTerm); + out.print("conf: ") // + .println(_conf); + out.print("targetPriority: ") // + .println(_targetPriority); + + // timers + out.println("electionTimer: "); + this.electionTimer.describe(out); + + out.println("voteTimer: "); + this.voteTimer.describe(out); + + out.println("stepDownTimer: "); + this.stepDownTimer.describe(out); + + out.println("snapshotTimer: "); + this.snapshotTimer.describe(out); + + // logManager + out.println("logManager: "); + this.logManager.describe(out); + + // fsmCaller + out.println("fsmCaller: "); + this.fsmCaller.describe(out); + + // ballotBox + out.println("ballotBox: "); + this.ballotBox.describe(out); + + // snapshotExecutor + if (this.snapshotExecutor != null) { + out.println("snapshotExecutor: "); + this.snapshotExecutor.describe(out); + } + + // replicators + out.println("replicatorGroup: "); + this.replicatorGroup.describe(out); + + // log storage + if (this.logStorage instanceof Describer) { + out.println("logStorage: "); + ((Describer) this.logStorage).describe(out); + } + } + + @Override + public String toString() { + return "JRaftNode [nodeId=" + getNodeId() + "]"; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeMetrics.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeMetrics.java new file mode 100644 index 0000000..8867224 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/NodeMetrics.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; + +/** + * Node metrics + * + * @author dennis + */ +public class NodeMetrics { + + private final MetricRegistry metrics; + + public NodeMetrics(final boolean enableMetrics) { + if (enableMetrics) { + this.metrics = new MetricRegistry(); + } else { + this.metrics = null; + } + } + + /** + * Retrieve the metrics map, returns empty map if it is disabled. + * + * @return metrics map + */ + public Map getMetrics() { + if (this.metrics != null) { + return this.metrics.getMetrics(); + } + return Collections.emptyMap(); + } + + /** + * Retrieve the metrics registry, return null if is is disabled. + * + * @return metrics registry + */ + public MetricRegistry getMetricRegistry() { + return this.metrics; + } + + /** + * Whether metric is enabled. + * + * @return true if metric is enabled + */ + public boolean isEnabled() { + return this.metrics != null; + } + + /** + * Records operation times. + * @param key key of operation + * @param times times of operation + */ + public void recordTimes(final String key, final long times) { + if (this.metrics != null) { + this.metrics.counter(key).inc(times); + } + } + + /** + * Records operation batch size. + * + * @param key key of operation + * @param size size of operation + */ + public void recordSize(final String key, final long size) { + if (this.metrics != null) { + this.metrics.histogram(key).update(size); + } + } + + /** + * Records operation latency. + * + * @param key key of operation + * @param duration duration of operation + */ + public void recordLatency(final String key, final long duration) { + if (this.metrics != null) { + this.metrics.timer(key).update(duration, TimeUnit.MILLISECONDS); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReadOnlyServiceImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReadOnlyServiceImpl.java new file mode 100644 index 0000000..8d7fe73 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReadOnlyServiceImpl.java @@ -0,0 +1,451 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.FSMCaller.LastAppliedLogIndexListener; +import com.alipay.sofa.jraft.ReadOnlyService; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.entity.ReadIndexState; +import com.alipay.sofa.jraft.entity.ReadIndexStatus; +import com.alipay.sofa.jraft.error.OverloadException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.option.ReadOnlyServiceOptions; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter; +import com.alipay.sofa.jraft.util.Bytes; +import com.alipay.sofa.jraft.util.DisruptorBuilder; +import com.alipay.sofa.jraft.util.DisruptorMetricSet; +import com.alipay.sofa.jraft.util.LogExceptionHandler; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Utils; +import com.google.protobuf.ZeroByteStringHelper; +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.EventTranslator; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +/** + * Read-only service implementation. + * + * @author dennis + */ +public class ReadOnlyServiceImpl implements ReadOnlyService, LastAppliedLogIndexListener { + + /** Disruptor to run readonly service. */ + private Disruptor readIndexDisruptor; + private RingBuffer readIndexQueue; + private RaftOptions raftOptions; + private NodeImpl node; + private final Lock lock = new ReentrantLock(); + private FSMCaller fsmCaller; + private volatile CountDownLatch shutdownLatch; + + private ScheduledExecutorService scheduledExecutorService; + + private NodeMetrics nodeMetrics; + + private volatile RaftException error; + + // + private final TreeMap> pendingNotifyStatus = new TreeMap<>(); + + private static final Logger LOG = LoggerFactory + .getLogger(ReadOnlyServiceImpl.class); + + private static class ReadIndexEvent { + Bytes requestContext; + ReadIndexClosure done; + CountDownLatch shutdownLatch; + long startTime; + + private void reset() { + this.requestContext = null; + this.done = null; + this.shutdownLatch = null; + this.startTime = 0L; + } + } + + private static class ReadIndexEventFactory implements EventFactory { + + @Override + public ReadIndexEvent newInstance() { + return new ReadIndexEvent(); + } + } + + private class ReadIndexEventHandler implements EventHandler { + // task list for batch + private final List events = new ArrayList<>( + ReadOnlyServiceImpl.this.raftOptions.getApplyBatch()); + + @Override + public void onEvent(final ReadIndexEvent newEvent, final long sequence, final boolean endOfBatch) + throws Exception { + if (newEvent.shutdownLatch != null) { + executeReadIndexEvents(this.events); + reset(); + newEvent.shutdownLatch.countDown(); + return; + } + + this.events.add(newEvent); + if (this.events.size() >= ReadOnlyServiceImpl.this.raftOptions.getApplyBatch() || endOfBatch) { + executeReadIndexEvents(this.events); + reset(); + } + } + + private void reset() { + for (final ReadIndexEvent event : this.events) { + event.reset(); + } + this.events.clear(); + } + } + + /** + * ReadIndexResponse process closure + * + * @author dennis + */ + class ReadIndexResponseClosure extends RpcResponseClosureAdapter { + + final List states; + final ReadIndexRequest request; + + public ReadIndexResponseClosure(final List states, final ReadIndexRequest request) { + super(); + this.states = states; + this.request = request; + } + + /** + * Called when ReadIndex response returns. + */ + @Override + public void run(final Status status) { + if (!status.isOk()) { + notifyFail(status); + return; + } + final ReadIndexResponse readIndexResponse = getResponse(); + if (!readIndexResponse.getSuccess()) { + notifyFail(new Status(-1, "Fail to run ReadIndex task, maybe the leader stepped down.")); + return; + } + // Success + final ReadIndexStatus readIndexStatus = new ReadIndexStatus(this.states, this.request, + readIndexResponse.getIndex()); + for (final ReadIndexState state : this.states) { + // Records current commit log index. + state.setIndex(readIndexResponse.getIndex()); + } + + boolean doUnlock = true; + ReadOnlyServiceImpl.this.lock.lock(); + try { + if (readIndexStatus.isApplied(ReadOnlyServiceImpl.this.fsmCaller.getLastAppliedIndex())) { + // Already applied, notify readIndex request. + ReadOnlyServiceImpl.this.lock.unlock(); + doUnlock = false; + notifySuccess(readIndexStatus); + } else { + if (readIndexStatus.isOverMaxReadIndexLag(ReadOnlyServiceImpl.this.fsmCaller.getLastAppliedIndex(), ReadOnlyServiceImpl.this.raftOptions.getMaxReadIndexLag())) { + ReadOnlyServiceImpl.this.lock.unlock(); + doUnlock = false; + notifyFail(new Status(-1, "Fail to run ReadIndex task, the gap of current node's apply index between leader's commit index over maxReadIndexLag")); + } else { + // Not applied, add it to pending-notify cache. + ReadOnlyServiceImpl.this.pendingNotifyStatus + .computeIfAbsent(readIndexStatus.getIndex(), k -> new ArrayList<>(10)) // + .add(readIndexStatus); + } + } + } finally { + if (doUnlock) { + ReadOnlyServiceImpl.this.lock.unlock(); + } + } + } + + private void notifyFail(final Status status) { + final long nowMs = Utils.monotonicMs(); + for (final ReadIndexState state : this.states) { + ReadOnlyServiceImpl.this.nodeMetrics.recordLatency("read-index", nowMs - state.getStartTimeMs()); + final ReadIndexClosure done = state.getDone(); + if (done != null) { + final Bytes reqCtx = state.getRequestContext(); + done.run(status, ReadIndexClosure.INVALID_LOG_INDEX, reqCtx != null ? reqCtx.get() : null); + } + } + } + } + + private void executeReadIndexEvents(final List events) { + if (events.isEmpty()) { + return; + } + final ReadIndexRequest.Builder rb = ReadIndexRequest.newBuilder() // + .setGroupId(this.node.getGroupId()) // + .setServerId(this.node.getServerId().toString()); + + final List states = new ArrayList<>(events.size()); + + for (final ReadIndexEvent event : events) { + rb.addEntries(ZeroByteStringHelper.wrap(event.requestContext.get())); + states.add(new ReadIndexState(event.requestContext, event.done, event.startTime)); + } + final ReadIndexRequest request = rb.build(); + + this.node.handleReadIndexRequest(request, new ReadIndexResponseClosure(states, request)); + } + + private void resetPendingStatusError(final Status st) { + this.lock.lock(); + try { + final Iterator> it = this.pendingNotifyStatus.values().iterator(); + while (it.hasNext()) { + final List statuses = it.next(); + for (final ReadIndexStatus status : statuses) { + reportError(status, st); + } + it.remove(); + } + } finally { + this.lock.unlock(); + } + } + + @Override + public boolean init(final ReadOnlyServiceOptions opts) { + this.node = opts.getNode(); + this.nodeMetrics = this.node.getNodeMetrics(); + this.fsmCaller = opts.getFsmCaller(); + this.raftOptions = opts.getRaftOptions(); + + this.scheduledExecutorService = Executors + .newSingleThreadScheduledExecutor(new NamedThreadFactory("ReadOnlyService-PendingNotify-Scanner", true)); + this.readIndexDisruptor = DisruptorBuilder. newInstance() // + .setEventFactory(new ReadIndexEventFactory()) // + .setRingBufferSize(this.raftOptions.getDisruptorBufferSize()) // + .setThreadFactory(new NamedThreadFactory("JRaft-ReadOnlyService-Disruptor-", true)) // + .setWaitStrategy(new BlockingWaitStrategy()) // + .setProducerType(ProducerType.MULTI) // + .build(); + this.readIndexDisruptor.handleEventsWith(new ReadIndexEventHandler()); + this.readIndexDisruptor + .setDefaultExceptionHandler(new LogExceptionHandler(getClass().getSimpleName())); + this.readIndexQueue = this.readIndexDisruptor.start(); + if (this.nodeMetrics.getMetricRegistry() != null) { + this.nodeMetrics.getMetricRegistry() // + .register("jraft-read-only-service-disruptor", new DisruptorMetricSet(this.readIndexQueue)); + } + // listen on lastAppliedLogIndex change events. + this.fsmCaller.addLastAppliedLogIndexListener(this); + + // start scanner + this.scheduledExecutorService.scheduleAtFixedRate(() -> onApplied(this.fsmCaller.getLastAppliedIndex()), + this.raftOptions.getMaxElectionDelayMs(), this.raftOptions.getMaxElectionDelayMs(), TimeUnit.MILLISECONDS); + return true; + } + + @Override + public synchronized void setError(final RaftException error) { + if (this.error == null) { + this.error = error; + } + } + + @Override + public synchronized void shutdown() { + if (this.shutdownLatch != null) { + return; + } + this.shutdownLatch = new CountDownLatch(1); + Utils.runInThread( // + () -> this.readIndexQueue.publishEvent((event, sequence) -> event.shutdownLatch = this.shutdownLatch)); + this.scheduledExecutorService.shutdown(); + } + + @Override + public void join() throws InterruptedException { + if (this.shutdownLatch != null) { + this.shutdownLatch.await(); + } + this.readIndexDisruptor.shutdown(); + resetPendingStatusError(new Status(RaftError.ESTOP, "Node is quit.")); + this.scheduledExecutorService.awaitTermination(5, TimeUnit.SECONDS); + } + + @Override + public void addRequest(final byte[] reqCtx, final ReadIndexClosure closure) { + if (this.shutdownLatch != null) { + Utils.runClosureInThread(closure, new Status(RaftError.EHOSTDOWN, "Was stopped")); + throw new IllegalStateException("Service already shutdown."); + } + try { + EventTranslator translator = (event, sequence) -> { + event.done = closure; + event.requestContext = new Bytes(reqCtx); + event.startTime = Utils.monotonicMs(); + }; + + switch(this.node.getOptions().getApplyTaskMode()) { + case Blocking: + this.readIndexQueue.publishEvent(translator); + break; + case NonBlocking: + default: + if (!this.readIndexQueue.tryPublishEvent(translator)) { + final String errorMsg = "Node is busy, has too many read-index requests, queue is full and bufferSize="+ this.readIndexQueue.getBufferSize(); + Utils.runClosureInThread(closure, + new Status(RaftError.EBUSY, errorMsg)); + this.nodeMetrics.recordTimes("read-index-overload-times", 1); + LOG.warn("Node {} ReadOnlyServiceImpl readIndexQueue is overload.", this.node.getNodeId()); + if(closure == null) { + throw new OverloadException(errorMsg); + } + } + break; + } + } catch (final Exception e) { + Utils.runClosureInThread(closure, new Status(RaftError.EPERM, "Node is down.")); + } + } + + /** + * Called when lastAppliedIndex updates. + * + * @param appliedIndex applied index + */ + @Override + public void onApplied(final long appliedIndex) { + // TODO reuse pendingStatuses list? + List pendingStatuses = null; + this.lock.lock(); + try { + if (this.pendingNotifyStatus.isEmpty()) { + return; + } + // Find all statuses that log index less than or equal to appliedIndex. + final Map> statuses = this.pendingNotifyStatus.headMap(appliedIndex, true); + if (statuses != null) { + pendingStatuses = new ArrayList<>(statuses.size() << 1); + + final Iterator>> it = statuses.entrySet().iterator(); + while (it.hasNext()) { + final Map.Entry> entry = it.next(); + pendingStatuses.addAll(entry.getValue()); + // Remove the entry from statuses, it will also be removed in pendingNotifyStatus. + it.remove(); + } + + } + + /* + * Remaining pending statuses are notified by error if it is presented. + * When the node is in error state, consider following situations: + * 1. If commitIndex > appliedIndex, then all pending statuses should be notified by error status. + * 2. When commitIndex == appliedIndex, there will be no more pending statuses. + */ + if (this.error != null) { + resetPendingStatusError(this.error.getStatus()); + } + } finally { + this.lock.unlock(); + if (pendingStatuses != null && !pendingStatuses.isEmpty()) { + for (final ReadIndexStatus status : pendingStatuses) { + notifySuccess(status); + } + } + } + } + + /** + * Flush all events in disruptor. + */ + @OnlyForTest + void flush() throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + this.readIndexQueue.publishEvent((task, sequence) -> task.shutdownLatch = latch); + latch.await(); + } + + @OnlyForTest + TreeMap> getPendingNotifyStatus() { + return this.pendingNotifyStatus; + } + + @OnlyForTest + RaftOptions getRaftOptions() { + return this.raftOptions; + } + + private void reportError(final ReadIndexStatus status, final Status st) { + final long nowMs = Utils.monotonicMs(); + final List states = status.getStates(); + final int taskCount = states.size(); + for (int i = 0; i < taskCount; i++) { + final ReadIndexState task = states.get(i); + final ReadIndexClosure done = task.getDone(); + if (done != null) { + this.nodeMetrics.recordLatency("read-index", nowMs - task.getStartTimeMs()); + done.run(st); + } + } + } + + private void notifySuccess(final ReadIndexStatus status) { + final long nowMs = Utils.monotonicMs(); + final List states = status.getStates(); + final int taskCount = states.size(); + for (int i = 0; i < taskCount; i++) { + final ReadIndexState task = states.get(i); + final ReadIndexClosure done = task.getDone(); // stack copy + if (done != null) { + this.nodeMetrics.recordLatency("read-index", nowMs - task.getStartTimeMs()); + done.setResult(task.getIndex(), task.getRequestContext().get()); + done.run(Status.OK()); + } + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Replicator.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Replicator.java new file mode 100644 index 0000000..1dd9b12 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Replicator.java @@ -0,0 +1,1891 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.nio.ByteBuffer; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import javax.annotation.concurrent.ThreadSafe; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.CatchUpClosure; +import com.alipay.sofa.jraft.core.Replicator.ReplicatorStateListener.ReplicatorState; +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.RaftOutter; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.option.ReplicatorOptions; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter; +import com.alipay.sofa.jraft.rpc.RpcUtils; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.util.ByteBufferCollector; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Recyclable; +import com.alipay.sofa.jraft.util.RecyclableByteBufferList; +import com.alipay.sofa.jraft.util.RecycleUtil; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.ThreadId; +import com.alipay.sofa.jraft.util.Utils; +import com.alipay.sofa.jraft.util.internal.ThrowUtil; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricFilter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.MetricSet; +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import com.google.protobuf.ZeroByteStringHelper; + +import static com.codahale.metrics.MetricRegistry.name; + +/** + * Replicator for replicating log entry from leader to followers. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 10:32:02 AM + */ +@ThreadSafe +public class Replicator implements ThreadId.OnError { + + private static final Logger LOG = LoggerFactory.getLogger(Replicator.class); + + private final RaftClientService rpcService; + // Next sending log index + private volatile long nextIndex; + private int consecutiveErrorTimes = 0; + private boolean hasSucceeded; + private long timeoutNowIndex; + private volatile long lastRpcSendTimestamp; + private volatile long heartbeatCounter = 0; + private volatile long probeCounter = 0; + private volatile long appendEntriesCounter = 0; + private volatile long installSnapshotCounter = 0; + protected Stat statInfo = new Stat(); + private ScheduledFuture blockTimer; + + // Cached the latest RPC in-flight request. + private Inflight rpcInFly; + // Heartbeat RPC future + private Future heartbeatInFly; + // Timeout request RPC future + private Future timeoutNowInFly; + // In-flight RPC requests, FIFO queue + private final ArrayDeque inflights = new ArrayDeque<>(); + + private long waitId = -1L; + protected ThreadId id; + private final ReplicatorOptions options; + private final RaftOptions raftOptions; + + private ScheduledFuture heartbeatTimer; + private volatile SnapshotReader reader; + private CatchUpClosure catchUpClosure; + private final Scheduler timerManager; + private final NodeMetrics nodeMetrics; + private volatile State state; + + // Request sequence + private int reqSeq = 0; + // Response sequence + private int requiredNextSeq = 0; + // Replicator state reset version + private int version = 0; + + // Pending response queue; + private final PriorityQueue pendingResponses = new PriorityQueue<>(50); + + private final String metricName; + + private int getAndIncrementReqSeq() { + final int prev = this.reqSeq; + this.reqSeq++; + if (this.reqSeq < 0) { + this.reqSeq = 0; + } + return prev; + } + + private int getAndIncrementRequiredNextSeq() { + final int prev = this.requiredNextSeq; + this.requiredNextSeq++; + if (this.requiredNextSeq < 0) { + this.requiredNextSeq = 0; + } + return prev; + } + + /** + * Replicator internal state + * @author dennis + * + */ + public enum State { + Created, + Probe, // probe follower state + Snapshot, // installing snapshot to follower + Replicate, // replicate logs normally + Destroyed // destroyed + } + + public Replicator(final ReplicatorOptions replicatorOptions, final RaftOptions raftOptions) { + super(); + this.options = replicatorOptions; + this.nodeMetrics = this.options.getNode().getNodeMetrics(); + this.nextIndex = this.options.getLogManager().getLastLogIndex() + 1; + this.timerManager = replicatorOptions.getTimerManager(); + this.raftOptions = raftOptions; + this.rpcService = replicatorOptions.getRaftRpcService(); + this.metricName = getReplicatorMetricName(replicatorOptions); + setState(State.Created); + } + + /** + * Replicator metric set. + * @author dennis + * + */ + private static final class ReplicatorMetricSet implements MetricSet { + private final ReplicatorOptions opts; + private final Replicator r; + + private ReplicatorMetricSet(final ReplicatorOptions opts, final Replicator r) { + this.opts = opts; + this.r = r; + } + + @Override + public Map getMetrics() { + final Map gauges = new HashMap<>(); + gauges.put("log-lags", + (Gauge) () -> this.opts.getLogManager().getLastLogIndex() - (this.r.nextIndex - 1)); + gauges.put("next-index", (Gauge) () -> this.r.nextIndex); + gauges.put("heartbeat-times", (Gauge) () -> this.r.heartbeatCounter); + gauges.put("install-snapshot-times", (Gauge) () -> this.r.installSnapshotCounter); + gauges.put("probe-times", (Gauge) () -> this.r.probeCounter); + gauges.put("append-entries-times", (Gauge) () -> this.r.appendEntriesCounter); + return gauges; + } + } + + /** + * Internal state + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 10:38:45 AM + */ + enum RunningState { + IDLE, // idle + BLOCKING, // blocking state + APPENDING_ENTRIES, // appending log entries + INSTALLING_SNAPSHOT // installing snapshot + } + + enum ReplicatorEvent { + CREATED, // created + ERROR, // error + DESTROYED ,// destroyed + STATE_CHANGED; // state changed. + } + + /** + * User can implement the ReplicatorStateListener interface by themselves. + * So they can do some their own logic codes when replicator created, destroyed or had some errors. + * + * @author zongtanghu + * + * 2019-Aug-20 2:32:10 PM + */ + public interface ReplicatorStateListener { + + /** + * Represents state changes in the replicator. + * @author boyan(boyan@antfin.com) + * + */ + enum ReplicatorState{ + /** + * The replicator is created. + */ + CREATED, + /** + * The replicator is destroyed. + */ + DESTROYED, + /** + * The replicator begins to doing it's job(replicating logs or installing snapshot). + */ + ONLINE, + /** + * The replicaotr is suspended by raft error or lost connection. + */ + OFFLINE + } + + + /** + * Called when this replicator has been created. + * + * @param peer replicator related peerId + */ + void onCreated(final PeerId peer); + + /** + * Called when this replicator has some errors. + * + * @param peer replicator related peerId + * @param status replicator's error detailed status + */ + void onError(final PeerId peer, final Status status); + + /** + * Called when this replicator has been destroyed. + * + * @param peer replicator related peerId + */ + void onDestroyed(final PeerId peer); + + /** + * Called when the replicator state is changed. See {@link ReplicatorState} + * @param peer the replicator's peer id. + * @param newState the new replicator state. + * @since 1.3.6 + */ + default void stateChanged(final PeerId peer, final ReplicatorState newState) {} + } + + private static void notifyReplicatorStatusListener(final Replicator replicator, final ReplicatorEvent event, + final Status status) { + notifyReplicatorStatusListener(replicator, event, status, null); + } + + /** + * Notify replicator event(such as created, error, destroyed) to replicatorStateListener which is implemented by users. + * + * @param replicator replicator object + * @param event replicator's state listener event type + * @param status replicator's error detailed status + */ + private static void notifyReplicatorStatusListener(final Replicator replicator, final ReplicatorEvent event, + final Status status, final ReplicatorState newState) { + final ReplicatorOptions replicatorOpts = Requires.requireNonNull(replicator.getOpts(), "replicatorOptions"); + final Node node = Requires.requireNonNull(replicatorOpts.getNode(), "node"); + final PeerId peer = Requires.requireNonNull(replicatorOpts.getPeerId(), "peer"); + + final List listenerList = node.getReplicatorStatueListeners(); + for (int i = 0; i < listenerList.size(); i++) { + final ReplicatorStateListener listener = listenerList.get(i); + if (listener != null) { + try { + switch (event) { + case CREATED: + RpcUtils.runInThread(() -> listener.onCreated(peer)); + break; + case ERROR: + RpcUtils.runInThread(() -> listener.onError(peer, status)); + break; + case DESTROYED: + RpcUtils.runInThread(() -> listener.onDestroyed(peer)); + break; + case STATE_CHANGED: + RpcUtils.runInThread(() -> listener.stateChanged(peer, newState)); + default: + break; + } + } catch (final Exception e) { + LOG.error("Fail to notify ReplicatorStatusListener, listener={}, event={}.", listener, event); + } + } + } + } + + /** + * Notify replicator event(such as created, error, destroyed) to replicatorStateListener which is implemented by users for none status. + * + * @param replicator replicator object + * @param event replicator's state listener event type + */ + private static void notifyReplicatorStatusListener(final Replicator replicator, final ReplicatorEvent event) { + notifyReplicatorStatusListener(replicator, event, null); + } + + /** + * Statistics structure + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 10:38:53 AM + */ + static class Stat { + RunningState runningState; + long firstLogIndex; + long lastLogIncluded; + long lastLogIndex; + long lastTermIncluded; + + @Override + public String toString() { + return ""; + } + + } + + // In-flight request type + enum RequestType { + Snapshot, // install snapshot + AppendEntries // replicate logs + } + + /** + * In-flight request. + * @author dennis + * + */ + static class Inflight { + // In-flight request count + final int count; + // Start log index + final long startIndex; + // Entries size in bytes + final int size; + // RPC future + final Future rpcFuture; + final RequestType requestType; + // Request sequence. + final int seq; + + public Inflight(final RequestType requestType, final long startIndex, final int count, final int size, + final int seq, final Future rpcFuture) { + super(); + this.seq = seq; + this.requestType = requestType; + this.count = count; + this.startIndex = startIndex; + this.size = size; + this.rpcFuture = rpcFuture; + } + + @Override + public String toString() { + return "Inflight [count=" + this.count + ", startIndex=" + this.startIndex + ", size=" + this.size + + ", rpcFuture=" + this.rpcFuture + ", requestType=" + this.requestType + ", seq=" + this.seq + "]"; + } + + boolean isSendingLogEntries() { + return this.requestType == RequestType.AppendEntries && this.count > 0; + } + } + + /** + * RPC response for AppendEntries/InstallSnapshot. + * @author dennis + * + */ + static class RpcResponse implements Comparable { + final Status status; + final Message request; + final Message response; + final long rpcSendTime; + final int seq; + final RequestType requestType; + + public RpcResponse(final RequestType reqType, final int seq, final Status status, final Message request, + final Message response, final long rpcSendTime) { + super(); + this.requestType = reqType; + this.seq = seq; + this.status = status; + this.request = request; + this.response = response; + this.rpcSendTime = rpcSendTime; + } + + @Override + public String toString() { + return "RpcResponse [status=" + this.status + ", request=" + this.request + ", response=" + this.response + + ", rpcSendTime=" + this.rpcSendTime + ", seq=" + this.seq + ", requestType=" + this.requestType + + "]"; + } + + /** + * Sort by sequence. + */ + @Override + public int compareTo(final RpcResponse o) { + return Integer.compare(this.seq, o.seq); + } + } + + @OnlyForTest + ArrayDeque getInflights() { + return this.inflights; + } + + State getState() { + return this.state; + } + + void setState(final State state) { + State oldState = this.state; + this.state = state; + + if(oldState != state) { + ReplicatorState newState = null; + switch(state) { + case Created: + newState = ReplicatorState.CREATED; + break; + case Replicate: + case Snapshot: + newState = ReplicatorState.ONLINE; + break; + case Probe: + newState = ReplicatorState.OFFLINE; + break; + case Destroyed: + newState = ReplicatorState.DESTROYED; + break; + } + + if(newState != null) { + notifyReplicatorStatusListener(this, ReplicatorEvent.STATE_CHANGED, null, newState); + } + } + } + + @OnlyForTest + int getReqSeq() { + return this.reqSeq; + } + + @OnlyForTest + int getRequiredNextSeq() { + return this.requiredNextSeq; + } + + @OnlyForTest + int getVersion() { + return this.version; + } + + @OnlyForTest + public PriorityQueue getPendingResponses() { + return this.pendingResponses; + } + + @OnlyForTest + long getWaitId() { + return this.waitId; + } + + @OnlyForTest + ScheduledFuture getBlockTimer() { + return this.blockTimer; + } + + @OnlyForTest + long getTimeoutNowIndex() { + return this.timeoutNowIndex; + } + + @OnlyForTest + ReplicatorOptions getOpts() { + return this.options; + } + + @OnlyForTest + long getRealNextIndex() { + return this.nextIndex; + } + + @OnlyForTest + Future getRpcInFly() { + if (this.rpcInFly == null) { + return null; + } + return this.rpcInFly.rpcFuture; + } + + @OnlyForTest + Future getHeartbeatInFly() { + return this.heartbeatInFly; + } + + @OnlyForTest + ScheduledFuture getHeartbeatTimer() { + return this.heartbeatTimer; + } + + @OnlyForTest + void setHasSucceeded() { + this.hasSucceeded = true; + } + + @OnlyForTest + Future getTimeoutNowInFly() { + return this.timeoutNowInFly; + } + + /** + * Adds a in-flight request + * @param reqType type of request + * @param count count if request + * @param size size in bytes + */ + private void addInflight(final RequestType reqType, final long startIndex, final int count, final int size, + final int seq, final Future rpcInfly) { + this.rpcInFly = new Inflight(reqType, startIndex, count, size, seq, rpcInfly); + this.inflights.add(this.rpcInFly); + this.nodeMetrics.recordSize(name(this.metricName, "replicate-inflights-count"), this.inflights.size()); + } + + /** + * Returns the next in-flight sending index, return -1 when can't send more in-flight requests. + * + * @return next in-flight sending index + */ + long getNextSendIndex() { + // Fast path + if (this.inflights.isEmpty()) { + return this.nextIndex; + } + // Too many in-flight requests. + if (this.inflights.size() > this.raftOptions.getMaxReplicatorInflightMsgs()) { + return -1L; + } + // Last request should be a AppendEntries request and has some entries. + if (this.rpcInFly != null && this.rpcInFly.isSendingLogEntries()) { + return this.rpcInFly.startIndex + this.rpcInFly.count; + } + return -1L; + } + + private Inflight pollInflight() { + return this.inflights.poll(); + } + + private void startHeartbeatTimer(final long startMs) { + final long dueTime = startMs + this.options.getDynamicHeartBeatTimeoutMs(); + try { + this.heartbeatTimer = this.timerManager.schedule(() -> onTimeout(this.id), dueTime - Utils.nowMs(), + TimeUnit.MILLISECONDS); + } catch (final Exception e) { + LOG.error("Fail to schedule heartbeat timer", e); + onTimeout(this.id); + } + } + + void installSnapshot() { + if (getState() == State.Snapshot) { + LOG.warn("Replicator {} is installing snapshot, ignore the new request.", this.options.getPeerId()); + unlockId(); + return; + } + boolean doUnlock = true; + if (!this.rpcService.connect(this.options.getPeerId().getEndpoint())) { + LOG.error("Fail to check install snapshot connection to peer={}, give up to send install snapshot request.", this.options.getPeerId().getEndpoint()); + block(Utils.nowMs(), RaftError.EHOSTDOWN.getNumber()); + return; + } + try { + Requires.requireTrue(this.reader == null, + "Replicator %s already has a snapshot reader, current state is %s", this.options.getPeerId(), + getState()); + this.reader = this.options.getSnapshotStorage().open(); + if (this.reader == null) { + final NodeImpl node = this.options.getNode(); + final RaftException error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT); + error.setStatus(new Status(RaftError.EIO, "Fail to open snapshot")); + unlockId(); + doUnlock = false; + node.onError(error); + return; + } + final String uri = this.reader.generateURIForCopy(); + if (uri == null) { + final NodeImpl node = this.options.getNode(); + final RaftException error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT); + error.setStatus(new Status(RaftError.EIO, "Fail to generate uri for snapshot reader")); + releaseReader(); + unlockId(); + doUnlock = false; + node.onError(error); + return; + } + final RaftOutter.SnapshotMeta meta = this.reader.load(); + if (meta == null) { + final String snapshotPath = this.reader.getPath(); + final NodeImpl node = this.options.getNode(); + final RaftException error = new RaftException(EnumOutter.ErrorType.ERROR_TYPE_SNAPSHOT); + error.setStatus(new Status(RaftError.EIO, "Fail to load meta from %s", snapshotPath)); + releaseReader(); + unlockId(); + doUnlock = false; + node.onError(error); + return; + } + final InstallSnapshotRequest.Builder rb = InstallSnapshotRequest.newBuilder(); + rb.setTerm(this.options.getTerm()); + rb.setGroupId(this.options.getGroupId()); + rb.setServerId(this.options.getServerId().toString()); + rb.setPeerId(this.options.getPeerId().toString()); + rb.setMeta(meta); + rb.setUri(uri); + + this.statInfo.runningState = RunningState.INSTALLING_SNAPSHOT; + this.statInfo.lastLogIncluded = meta.getLastIncludedIndex(); + this.statInfo.lastTermIncluded = meta.getLastIncludedTerm(); + + final InstallSnapshotRequest request = rb.build(); + setState(State.Snapshot); + // noinspection NonAtomicOperationOnVolatileField + this.installSnapshotCounter++; + final long monotonicSendTimeMs = Utils.monotonicMs(); + final int stateVersion = this.version; + final int seq = getAndIncrementReqSeq(); + final Future rpcFuture = this.rpcService.installSnapshot(this.options.getPeerId().getEndpoint(), + request, new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + onRpcReturned(Replicator.this.id, RequestType.Snapshot, status, request, getResponse(), seq, + stateVersion, monotonicSendTimeMs); + } + }); + addInflight(RequestType.Snapshot, this.nextIndex, 0, 0, seq, rpcFuture); + } finally { + if (doUnlock) { + unlockId(); + } + } + } + + @SuppressWarnings("unused") + static boolean onInstallSnapshotReturned(final ThreadId id, final Replicator r, final Status status, + final InstallSnapshotRequest request, + final InstallSnapshotResponse response) { + boolean success = true; + r.releaseReader(); + // noinspection ConstantConditions + do { + final StringBuilder sb = new StringBuilder("Node "). // + append(r.options.getGroupId()).append(":").append(r.options.getServerId()). // + append(" received InstallSnapshotResponse from ").append(r.options.getPeerId()). // + append(" lastIncludedIndex=").append(request.getMeta().getLastIncludedIndex()). // + append(" lastIncludedTerm=").append(request.getMeta().getLastIncludedTerm()); + if (!status.isOk()) { + sb.append(" error:").append(status); + LOG.info(sb.toString()); + notifyReplicatorStatusListener(r, ReplicatorEvent.ERROR, status); + if (++r.consecutiveErrorTimes % 10 == 0) { + LOG.warn("Fail to install snapshot at peer={}, error={}", r.options.getPeerId(), status); + } + success = false; + break; + } + if (!response.getSuccess()) { + sb.append(" success=false"); + LOG.info(sb.toString()); + success = false; + break; + } + // success + r.nextIndex = request.getMeta().getLastIncludedIndex() + 1; + sb.append(" success=true"); + LOG.info(sb.toString()); + } while (false); + // We don't retry installing the snapshot explicitly. + // id is unlock in sendEntries + if (!success) { + //should reset states + r.resetInflights(); + r.setState(State.Probe); + r.block(Utils.nowMs(), status.getCode()); + return false; + } + r.hasSucceeded = true; + r.notifyOnCaughtUp(RaftError.SUCCESS.getNumber(), false); + if (r.timeoutNowIndex > 0 && r.timeoutNowIndex < r.nextIndex) { + r.sendTimeoutNow(false, false); + } + // id is unlock in _send_entriesheartbeatCounter + r.setState(State.Replicate); + return true; + } + + private void sendEmptyEntries(final boolean isHeartbeat) { + sendEmptyEntries(isHeartbeat, null); + } + + /** + * Send probe or heartbeat request + * @param isHeartbeat if current entries is heartbeat + * @param heartBeatClosure heartbeat callback + */ + @SuppressWarnings("NonAtomicOperationOnVolatileField") + private void sendEmptyEntries(final boolean isHeartbeat, + final RpcResponseClosure heartBeatClosure) { + final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder(); + if (!fillCommonFields(rb, this.nextIndex - 1, isHeartbeat)) { + // id is unlock in installSnapshot + installSnapshot(); + if (isHeartbeat && heartBeatClosure != null) { + RpcUtils.runClosureInThread(heartBeatClosure, new Status(RaftError.EAGAIN, + "Fail to send heartbeat to peer %s", this.options.getPeerId())); + } + return; + } + try { + final long monotonicSendTimeMs = Utils.monotonicMs(); + + if (isHeartbeat) { + final AppendEntriesRequest request = rb.build(); + // Sending a heartbeat request + this.heartbeatCounter++; + RpcResponseClosure heartbeatDone; + // Prefer passed-in closure. + if (heartBeatClosure != null) { + heartbeatDone = heartBeatClosure; + } else { + heartbeatDone = new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + onHeartbeatReturned(Replicator.this.id, status, request, getResponse(), monotonicSendTimeMs); + } + }; + } + this.heartbeatInFly = this.rpcService.appendEntries(this.options.getPeerId().getEndpoint(), request, + this.options.getElectionTimeoutMs() / 2, heartbeatDone); + } else { + // No entries and has empty data means a probe request. + // TODO(boyan) refactor, adds a new flag field? + rb.setData(ByteString.EMPTY); + final AppendEntriesRequest request = rb.build(); + // Sending a probe request. + this.statInfo.runningState = RunningState.APPENDING_ENTRIES; + this.statInfo.firstLogIndex = this.nextIndex; + this.statInfo.lastLogIndex = this.nextIndex - 1; + this.probeCounter++; + setState(State.Probe); + final int stateVersion = this.version; + final int seq = getAndIncrementReqSeq(); + final Future rpcFuture = this.rpcService.appendEntries(this.options.getPeerId().getEndpoint(), + request, -1, new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + onRpcReturned(Replicator.this.id, RequestType.AppendEntries, status, request, + getResponse(), seq, stateVersion, monotonicSendTimeMs); + } + + }); + + addInflight(RequestType.AppendEntries, this.nextIndex, 0, 0, seq, rpcFuture); + } + LOG.debug("Node {} send HeartbeatRequest to {} term {} lastCommittedIndex {}", this.options.getNode() + .getNodeId(), this.options.getPeerId(), this.options.getTerm(), rb.getCommittedIndex()); + } finally { + unlockId(); + } + } + + boolean prepareEntry(final long nextSendingIndex, final int offset, final RaftOutter.EntryMeta.Builder emb, + final RecyclableByteBufferList dateBuffer) { + if (dateBuffer.getCapacity() >= this.raftOptions.getMaxBodySize()) { + return false; + } + final long logIndex = nextSendingIndex + offset; + final LogEntry entry = this.options.getLogManager().getEntry(logIndex); + if (entry == null) { + return false; + } + emb.setTerm(entry.getId().getTerm()); + if (entry.hasChecksum()) { + emb.setChecksum(entry.getChecksum()); // since 1.2.6 + } + emb.setType(entry.getType()); + if (entry.getPeers() != null) { + Requires.requireTrue(!entry.getPeers().isEmpty(), "Empty peers at logIndex=%d", logIndex); + fillMetaPeers(emb, entry); + } else { + Requires.requireTrue(entry.getType() != EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION, + "Empty peers but is ENTRY_TYPE_CONFIGURATION type at logIndex=%d", logIndex); + } + final int remaining = entry.getData() != null ? entry.getData().remaining() : 0; + emb.setDataLen(remaining); + if (entry.getData() != null) { + // should slice entry data + dateBuffer.add(entry.getData().slice()); + } + return true; + } + + private void fillMetaPeers(final RaftOutter.EntryMeta.Builder emb, final LogEntry entry) { + for (final PeerId peer : entry.getPeers()) { + emb.addPeers(peer.toString()); + } + if (entry.getOldPeers() != null) { + for (final PeerId peer : entry.getOldPeers()) { + emb.addOldPeers(peer.toString()); + } + } + if (entry.getLearners() != null) { + for (final PeerId peer : entry.getLearners()) { + emb.addLearners(peer.toString()); + } + } + if (entry.getOldLearners() != null) { + for (final PeerId peer : entry.getOldLearners()) { + emb.addOldLearners(peer.toString()); + } + } + } + + public static ThreadId start(final ReplicatorOptions opts, final RaftOptions raftOptions) { + if (opts.getLogManager() == null || opts.getBallotBox() == null || opts.getNode() == null) { + throw new IllegalArgumentException("Invalid ReplicatorOptions."); + } + final Replicator r = new Replicator(opts, raftOptions); + if (!r.rpcService.connect(opts.getPeerId().getEndpoint())) { + LOG.error("Fail to init sending channel to {}.", opts.getPeerId()); + // Return and it will be retried later. + return null; + } + + // Register replicator metric set. + final MetricRegistry metricRegistry = opts.getNode().getNodeMetrics().getMetricRegistry(); + if (metricRegistry != null) { + try { + if (!metricRegistry.getNames().contains(r.metricName)) { + metricRegistry.register(r.metricName, new ReplicatorMetricSet(opts, r)); + } + } catch (final IllegalArgumentException e) { + // ignore + } + } + + // Start replication + r.id = new ThreadId(r, r); + r.id.lock(); + notifyReplicatorStatusListener(r, ReplicatorEvent.CREATED); + LOG.info("Replicator={}@{} is started", r.id, r.options.getPeerId()); + r.catchUpClosure = null; + r.lastRpcSendTimestamp = Utils.monotonicMs(); + r.startHeartbeatTimer(Utils.nowMs()); + // id.unlock in sendEmptyEntries + r.sendProbeRequest(); + return r.id; + } + + private String getReplicatorMetricName(final ReplicatorOptions opts) { + return "replicator-" + opts.getNode().getGroupId() + "/" + opts.getPeerId(); + } + + public static void waitForCaughtUp(final ThreadId id, final long maxMargin, final long dueTime, + final CatchUpClosure done) { + final Replicator r = (Replicator) id.lock(); + + if (r == null) { + RpcUtils.runClosureInThread(done, new Status(RaftError.EINVAL, "No such replicator")); + return; + } + try { + if (r.catchUpClosure != null) { + LOG.error("Previous wait_for_caught_up is not over"); + Utils.runClosureInThread(done, new Status(RaftError.EINVAL, "Duplicated call")); + return; + } + done.setMaxMargin(maxMargin); + if (dueTime > 0) { + done.setTimer(r.timerManager.schedule(() -> onCatchUpTimedOut(id), dueTime - Utils.nowMs(), + TimeUnit.MILLISECONDS)); + } + r.catchUpClosure = done; + } finally { + id.unlock(); + } + } + + @Override + public String toString() { + return "Replicator [state=" + getState() + ", statInfo=" + this.statInfo + ", peerId=" + + this.options.getPeerId() + ", type=" + this.options.getReplicatorType() + "]"; + } + + static void onBlockTimeoutInNewThread(final ThreadId id) { + if (id != null) { + continueSending(id, RaftError.ETIMEDOUT.getNumber()); + } + } + + /** + * Unblock and continue sending right now. + */ + static void unBlockAndSendNow(final ThreadId id) { + if (id == null) { + // It was destroyed already + return; + } + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return; + } + try { + if (r.blockTimer != null) { + if (r.blockTimer.cancel(true)) { + onBlockTimeout(id); + } + } + } finally { + id.unlock(); + } + } + + static boolean continueSending(final ThreadId id, final int errCode) { + if (id == null) { + //It was destroyed already + return true; + } + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return false; + } + r.waitId = -1; + if (errCode == RaftError.ETIMEDOUT.getNumber()) { + r.blockTimer = null; + // Send empty entries after block timeout to check the correct + // _next_index otherwise the replicator is likely waits in executor.shutdown(); + // _wait_more_entries and no further logs would be replicated even if the + // last_index of this followers is less than |next_index - 1| + r.sendProbeRequest(); + } else if (errCode != RaftError.ESTOP.getNumber()) { + // id is unlock in _send_entries + r.sendEntries(); + } else { + LOG.warn("Replicator {} stops sending entries.", id); + id.unlock(); + } + return true; + } + + static void onBlockTimeout(final ThreadId arg) { + RpcUtils.runInThread(() -> onBlockTimeoutInNewThread(arg)); + } + + void block(final long startTimeMs, @SuppressWarnings("unused") final int errorCode) { + // TODO: Currently we don't care about error_code which indicates why the + // very RPC fails. To make it better there should be different timeout for + // each individual error (e.g. we don't need check every + // heartbeat_timeout_ms whether a dead follower has come back), but it's just + // fine now. + if(this.blockTimer != null) { + // already in blocking state,return immediately. + unlockId(); + return; + } + final long dueTime = startTimeMs + this.options.getDynamicHeartBeatTimeoutMs(); + try { + LOG.debug("Blocking {} for {} ms", this.options.getPeerId(), this.options.getDynamicHeartBeatTimeoutMs()); + this.blockTimer = this.timerManager.schedule(() -> onBlockTimeout(this.id), dueTime - Utils.nowMs(), + TimeUnit.MILLISECONDS); + this.statInfo.runningState = RunningState.BLOCKING; + unlockId(); + } catch (final Exception e) { + this.blockTimer = null; + LOG.error("Fail to add timer", e); + // id unlock in sendEmptyEntries. + sendProbeRequest(); + } + } + + @Override + public void onError(final ThreadId id, final Object data, final int errorCode) { + final Replicator r = (Replicator) data; + if (errorCode == RaftError.ESTOP.getNumber()) { + try { + for (final Inflight inflight : r.inflights) { + if (inflight != r.rpcInFly) { + inflight.rpcFuture.cancel(true); + } + } + if (r.rpcInFly != null) { + r.rpcInFly.rpcFuture.cancel(true); + r.rpcInFly = null; + } + if (r.heartbeatInFly != null) { + r.heartbeatInFly.cancel(true); + r.heartbeatInFly = null; + } + if (r.timeoutNowInFly != null) { + r.timeoutNowInFly.cancel(true); + r.timeoutNowInFly = null; + } + if (r.heartbeatTimer != null) { + r.heartbeatTimer.cancel(true); + r.heartbeatTimer = null; + } + if (r.blockTimer != null) { + r.blockTimer.cancel(true); + r.blockTimer = null; + } + if (r.waitId >= 0) { + r.options.getLogManager().removeWaiter(r.waitId); + } + r.notifyOnCaughtUp(errorCode, true); + } finally { + r.destroy(); + } + } else if (errorCode == RaftError.ETIMEDOUT.getNumber()) { + RpcUtils.runInThread(() -> sendHeartbeat(id)); + } else { + // noinspection ConstantConditions + Requires.requireTrue(false, "Unknown error code for replicator: " + errorCode); + } + } + + private static void onCatchUpTimedOut(final ThreadId id) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return; + } + try { + r.notifyOnCaughtUp(RaftError.ETIMEDOUT.getNumber(), false); + } finally { + id.unlock(); + } + } + + private void notifyOnCaughtUp(final int code, final boolean beforeDestroy) { + if (this.catchUpClosure == null) { + return; + } + if (code != RaftError.ETIMEDOUT.getNumber()) { + if (this.nextIndex - 1 + this.catchUpClosure.getMaxMargin() < this.options.getLogManager() + .getLastLogIndex()) { + return; + } + if (this.catchUpClosure.isErrorWasSet()) { + return; + } + this.catchUpClosure.setErrorWasSet(true); + if (code != RaftError.SUCCESS.getNumber()) { + this.catchUpClosure.getStatus().setError(code, RaftError.describeCode(code)); + } + if (this.catchUpClosure.hasTimer()) { + if (!beforeDestroy && !this.catchUpClosure.getTimer().cancel(true)) { + // There's running timer task, let timer task trigger + // on_caught_up to void ABA problem + return; + } + } + } else { + // timed out + if (!this.catchUpClosure.isErrorWasSet()) { + this.catchUpClosure.getStatus().setError(code, RaftError.describeCode(code)); + } + } + final CatchUpClosure savedClosure = this.catchUpClosure; + this.catchUpClosure = null; + RpcUtils.runClosureInThread(savedClosure, savedClosure.getStatus()); + } + + private static void onTimeout(final ThreadId id) { + if (id != null) { + id.setError(RaftError.ETIMEDOUT.getNumber()); + } else { + LOG.warn("Replicator id is null when timeout, maybe it's destroyed."); + } + } + + void destroy() { + final ThreadId savedId = this.id; + LOG.info("Replicator {} is going to quit", savedId); + releaseReader(); + // Unregister replicator metric set + if (this.nodeMetrics.isEnabled()) { + this.nodeMetrics.getMetricRegistry() // + .removeMatching(MetricFilter.startsWith(this.metricName)); + } + setState(State.Destroyed); + notifyReplicatorStatusListener((Replicator) savedId.getData(), ReplicatorEvent.DESTROYED); + savedId.unlockAndDestroy(); + this.id = null; + } + + private void releaseReader() { + if (this.reader != null) { + Utils.closeQuietly(this.reader); + this.reader = null; + } + } + + static void onHeartbeatReturned(final ThreadId id, final Status status, final AppendEntriesRequest request, + final AppendEntriesResponse response, final long rpcSendTime) { + if (id == null) { + // replicator already was destroyed. + return; + } + final long startTimeMs = Utils.nowMs(); + Replicator r; + if ((r = (Replicator) id.lock()) == null) { + return; + } + boolean doUnlock = true; + try { + final boolean isLogDebugEnabled = LOG.isDebugEnabled(); + StringBuilder sb = null; + if (isLogDebugEnabled) { + sb = new StringBuilder("Node ") // + .append(r.options.getGroupId()) // + .append(':') // + .append(r.options.getServerId()) // + .append(" received HeartbeatResponse from ") // + .append(r.options.getPeerId()) // + .append(" prevLogIndex=") // + .append(request.getPrevLogIndex()) // + .append(" prevLogTerm=") // + .append(request.getPrevLogTerm()); + } + if (!status.isOk()) { + if (isLogDebugEnabled) { + sb.append(" fail, sleep, status=") // + .append(status); + LOG.debug(sb.toString()); + } + r.setState(State.Probe); + notifyReplicatorStatusListener(r, ReplicatorEvent.ERROR, status); + if (++r.consecutiveErrorTimes % 10 == 0) { + LOG.warn("Fail to issue RPC to {}, consecutiveErrorTimes={}, error={}", r.options.getPeerId(), + r.consecutiveErrorTimes, status); + } + r.startHeartbeatTimer(startTimeMs); + return; + } + r.consecutiveErrorTimes = 0; + if (response.getTerm() > r.options.getTerm()) { + if (isLogDebugEnabled) { + sb.append(" fail, greater term ") // + .append(response.getTerm()) // + .append(" expect term ") // + .append(r.options.getTerm()); + LOG.debug(sb.toString()); + } + final NodeImpl node = r.options.getNode(); + r.notifyOnCaughtUp(RaftError.EPERM.getNumber(), true); + r.destroy(); + node.increaseTermTo(response.getTerm(), new Status(RaftError.EHIGHERTERMRESPONSE, + "Leader receives higher term heartbeat_response from peer:%s", r.options.getPeerId())); + return; + } + if (!response.getSuccess() && response.hasLastLogIndex()) { + if (isLogDebugEnabled) { + sb.append(" fail, response term ") // + .append(response.getTerm()) // + .append(" lastLogIndex ") // + .append(response.getLastLogIndex()); + LOG.debug(sb.toString()); + } + LOG.warn("Heartbeat to peer {} failure, try to send a probe request.", r.options.getPeerId()); + doUnlock = false; + r.sendProbeRequest(); + r.startHeartbeatTimer(startTimeMs); + return; + } + if (isLogDebugEnabled) { + LOG.debug(sb.toString()); + } + if (rpcSendTime > r.lastRpcSendTimestamp) { + r.lastRpcSendTimestamp = rpcSendTime; + } + r.startHeartbeatTimer(startTimeMs); + } finally { + if (doUnlock) { + id.unlock(); + } + } + } + + @SuppressWarnings("ContinueOrBreakFromFinallyBlock") + static void onRpcReturned(final ThreadId id, final RequestType reqType, final Status status, final Message request, + final Message response, final int seq, final int stateVersion, final long rpcSendTime) { + if (id == null) { + return; + } + final long startTimeMs = Utils.nowMs(); + Replicator r; + if ((r = (Replicator) id.lock()) == null) { + return; + } + + if (stateVersion != r.version) { + LOG.debug( + "Replicator {} ignored old version response {}, current version is {}, request is {}\n, and response is {}\n, status is {}.", + r, stateVersion, r.version, request, response, status); + id.unlock(); + return; + } + + final PriorityQueue holdingQueue = r.pendingResponses; + holdingQueue.add(new RpcResponse(reqType, seq, status, request, response, rpcSendTime)); + + if (holdingQueue.size() > r.raftOptions.getMaxReplicatorInflightMsgs()) { + LOG.warn("Too many pending responses {} for replicator {}, maxReplicatorInflightMsgs={}", + holdingQueue.size(), r.options.getPeerId(), r.raftOptions.getMaxReplicatorInflightMsgs()); + r.resetInflights(); + r.setState(State.Probe); + r.sendProbeRequest(); + return; + } + + boolean continueSendEntries = false; + + final boolean isLogDebugEnabled = LOG.isDebugEnabled(); + StringBuilder sb = null; + if (isLogDebugEnabled) { + sb = new StringBuilder("Replicator ") // + .append(r) // + .append(" is processing RPC responses, "); + } + try { + int processed = 0; + while (!holdingQueue.isEmpty()) { + final RpcResponse queuedPipelinedResponse = holdingQueue.peek(); + + // Sequence mismatch, waiting for next response. + if (queuedPipelinedResponse.seq != r.requiredNextSeq) { + if (processed > 0) { + if (isLogDebugEnabled) { + sb.append("has processed ") // + .append(processed) // + .append(" responses, "); + } + break; + } else { + // Do not processed any responses, UNLOCK id and return. + continueSendEntries = false; + id.unlock(); + return; + } + } + holdingQueue.remove(); + processed++; + final Inflight inflight = r.pollInflight(); + if (inflight == null) { + // The previous in-flight requests were cleared. + if (isLogDebugEnabled) { + sb.append("ignore response because request not found: ") // + .append(queuedPipelinedResponse) // + .append(",\n"); + } + continue; + } + if (inflight.seq != queuedPipelinedResponse.seq) { + // reset state + LOG.warn( + "Replicator {} response sequence out of order, expect {}, but it is {}, reset state to try again.", + r, inflight.seq, queuedPipelinedResponse.seq); + r.resetInflights(); + r.setState(State.Probe); + continueSendEntries = false; + r.block(Utils.nowMs(), RaftError.EREQUEST.getNumber()); + return; + } + try { + switch (queuedPipelinedResponse.requestType) { + case AppendEntries: + continueSendEntries = onAppendEntriesReturned(id, inflight, queuedPipelinedResponse.status, + (AppendEntriesRequest) queuedPipelinedResponse.request, + (AppendEntriesResponse) queuedPipelinedResponse.response, rpcSendTime, startTimeMs, r); + break; + case Snapshot: + continueSendEntries = onInstallSnapshotReturned(id, r, queuedPipelinedResponse.status, + (InstallSnapshotRequest) queuedPipelinedResponse.request, + (InstallSnapshotResponse) queuedPipelinedResponse.response); + break; + } + } finally { + if (continueSendEntries) { + // Success, increase the response sequence. + r.getAndIncrementRequiredNextSeq(); + } else { + // The id is already unlocked in onAppendEntriesReturned/onInstallSnapshotReturned, we SHOULD break out. + break; + } + } + } + } finally { + if (isLogDebugEnabled) { + sb.append("after processed, continue to send entries: ") // + .append(continueSendEntries); + LOG.debug(sb.toString()); + } + if (continueSendEntries) { + // unlock in sendEntries. + r.sendEntries(); + } + } + } + + /** + * Reset in-flight state. + */ + void resetInflights() { + this.version++; + this.inflights.clear(); + this.pendingResponses.clear(); + final int rs = Math.max(this.reqSeq, this.requiredNextSeq); + this.reqSeq = this.requiredNextSeq = rs; + releaseReader(); + } + + private static boolean onAppendEntriesReturned(final ThreadId id, final Inflight inflight, final Status status, + final AppendEntriesRequest request, + final AppendEntriesResponse response, final long rpcSendTime, + final long startTimeMs, final Replicator r) { + if (inflight.startIndex != request.getPrevLogIndex() + 1) { + LOG.warn( + "Replicator {} received invalid AppendEntriesResponse, in-flight startIndex={}, request prevLogIndex={}, reset the replicator state and probe again.", + r, inflight.startIndex, request.getPrevLogIndex()); + r.resetInflights(); + r.setState(State.Probe); + // unlock id in sendEmptyEntries + r.sendProbeRequest(); + return false; + } + // record metrics + if (request.getEntriesCount() > 0) { + r.nodeMetrics.recordLatency("replicate-entries", Utils.monotonicMs() - rpcSendTime); + r.nodeMetrics.recordSize("replicate-entries-count", request.getEntriesCount()); + r.nodeMetrics.recordSize("replicate-entries-bytes", request.getData() != null ? request.getData().size() + : 0); + } + + final boolean isLogDebugEnabled = LOG.isDebugEnabled(); + StringBuilder sb = null; + if (isLogDebugEnabled) { + sb = new StringBuilder("Node ") // + .append(r.options.getGroupId()) // + .append(':') // + .append(r.options.getServerId()) // + .append(" received AppendEntriesResponse from ") // + .append(r.options.getPeerId()) // + .append(" prevLogIndex=") // + .append(request.getPrevLogIndex()) // + .append(" prevLogTerm=") // + .append(request.getPrevLogTerm()) // + .append(" count=") // + .append(request.getEntriesCount()); + } + if (!status.isOk()) { + // If the follower crashes, any RPC to the follower fails immediately, + // so we need to block the follower for a while instead of looping until + // it comes back or be removed + // dummy_id is unlock in block + if (isLogDebugEnabled) { + sb.append(" fail, sleep, status=") // + .append(status); + LOG.debug(sb.toString()); + } + notifyReplicatorStatusListener(r, ReplicatorEvent.ERROR, status); + if (++r.consecutiveErrorTimes % 10 == 0) { + LOG.warn("Fail to issue RPC to {}, consecutiveErrorTimes={}, error={}", r.options.getPeerId(), + r.consecutiveErrorTimes, status); + } + r.resetInflights(); + r.setState(State.Probe); + // unlock in in block + r.block(startTimeMs, status.getCode()); + return false; + } + r.consecutiveErrorTimes = 0; + if (!response.getSuccess()) { + // Target node is is busy, sleep for a while. + if(response.getErrorResponse().getErrorCode() == RaftError.EBUSY.getNumber()) { + if (isLogDebugEnabled) { + sb.append(" is busy, sleep, errorMsg='") // + .append(response.getErrorResponse().getErrorMsg()).append("'"); + LOG.debug(sb.toString()); + } + r.resetInflights(); + r.setState(State.Probe); + // unlock in in block + r.block(startTimeMs, status.getCode()); + return false; + } + + if (response.getTerm() > r.options.getTerm()) { + if (isLogDebugEnabled) { + sb.append(" fail, greater term ") // + .append(response.getTerm()) // + .append(" expect term ") // + .append(r.options.getTerm()); + LOG.debug(sb.toString()); + } + final NodeImpl node = r.options.getNode(); + r.notifyOnCaughtUp(RaftError.EPERM.getNumber(), true); + r.destroy(); + node.increaseTermTo(response.getTerm(), new Status(RaftError.EHIGHERTERMRESPONSE, + "Leader receives higher term heartbeat_response from peer:%s", r.options.getPeerId())); + return false; + } + if (isLogDebugEnabled) { + sb.append(" fail, find nextIndex remote lastLogIndex ").append(response.getLastLogIndex()) + .append(" local nextIndex ").append(r.nextIndex); + LOG.debug(sb.toString()); + } + if (rpcSendTime > r.lastRpcSendTimestamp) { + r.lastRpcSendTimestamp = rpcSendTime; + } + // Fail, reset the state to try again from nextIndex. + r.resetInflights(); + // prev_log_index and prev_log_term doesn't match + if (response.getLastLogIndex() + 1 < r.nextIndex) { + LOG.debug("LastLogIndex at peer={} is {}", r.options.getPeerId(), response.getLastLogIndex()); + // The peer contains less logs than leader + r.nextIndex = response.getLastLogIndex() + 1; + } else { + // The peer contains logs from old term which should be truncated, + // decrease _last_log_at_peer by one to test the right index to keep + if (r.nextIndex > 1) { + LOG.debug("logIndex={} dismatch", r.nextIndex); + r.nextIndex--; + } else { + LOG.error("Peer={} declares that log at index=0 doesn't match, which is not supposed to happen", + r.options.getPeerId()); + } + } + // dummy_id is unlock in _send_heartbeat + r.sendProbeRequest(); + return false; + } + if (isLogDebugEnabled) { + sb.append(", success"); + LOG.debug(sb.toString()); + } + // success + if (response.getTerm() != r.options.getTerm()) { + r.resetInflights(); + r.setState(State.Probe); + LOG.error("Fail, response term {} dismatch, expect term {}", response.getTerm(), r.options.getTerm()); + id.unlock(); + return false; + } + if (rpcSendTime > r.lastRpcSendTimestamp) { + r.lastRpcSendTimestamp = rpcSendTime; + } + final int entriesSize = request.getEntriesCount(); + if (entriesSize > 0) { + if (r.options.getReplicatorType().isFollower()) { + // Only commit index when the response is from follower. + r.options.getBallotBox().commitAt(r.nextIndex, r.nextIndex + entriesSize - 1, r.options.getPeerId()); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Replicated logs in [{}, {}] to peer {}", r.nextIndex, r.nextIndex + entriesSize - 1, + r.options.getPeerId()); + } + } + + r.setState(State.Replicate); + r.blockTimer = null; + r.nextIndex += entriesSize; + r.hasSucceeded = true; + r.notifyOnCaughtUp(RaftError.SUCCESS.getNumber(), false); + // dummy_id is unlock in _send_entries + if (r.timeoutNowIndex > 0 && r.timeoutNowIndex < r.nextIndex) { + r.sendTimeoutNow(false, false); + } + return true; + } + + private boolean fillCommonFields(final AppendEntriesRequest.Builder rb, long prevLogIndex, final boolean isHeartbeat) { + final long prevLogTerm = this.options.getLogManager().getTerm(prevLogIndex); + if (prevLogTerm == 0 && prevLogIndex != 0) { + if (!isHeartbeat) { + Requires.requireTrue(prevLogIndex < this.options.getLogManager().getFirstLogIndex()); + LOG.debug("logIndex={} was compacted", prevLogIndex); + return false; + } else { + // The log at prev_log_index has been compacted, which indicates + // we is or is going to install snapshot to the follower. So we let + // both prev_log_index and prev_log_term be 0 in the heartbeat + // request so that follower would do nothing besides updating its + // leader timestamp. + prevLogIndex = 0; + } + } + rb.setTerm(this.options.getTerm()); + rb.setGroupId(this.options.getGroupId()); + rb.setServerId(this.options.getServerId().toString()); + rb.setPeerId(this.options.getPeerId().toString()); + rb.setPrevLogIndex(prevLogIndex); + rb.setPrevLogTerm(prevLogTerm); + rb.setCommittedIndex(this.options.getBallotBox().getLastCommittedIndex()); + return true; + } + + private void waitMoreEntries(final long nextWaitIndex) { + try { + LOG.debug("Node {} waits more entries", this.options.getNode().getNodeId()); + if (this.waitId >= 0) { + return; + } + this.waitId = this.options.getLogManager().wait(nextWaitIndex - 1, + (arg, errorCode) -> continueSending((ThreadId) arg, errorCode), this.id); + this.statInfo.runningState = RunningState.IDLE; + } finally { + unlockId(); + } + } + + /** + * Send as many requests as possible. + */ + void sendEntries() { + boolean doUnlock = true; + try { + long prevSendIndex = -1; + while (true) { + final long nextSendingIndex = getNextSendIndex(); + if (nextSendingIndex > prevSendIndex) { + if (sendEntries(nextSendingIndex)) { + prevSendIndex = nextSendingIndex; + } else { + doUnlock = false; + // id already unlock in sendEntries when it returns false. + break; + } + } else { + break; + } + } + } finally { + if (doUnlock) { + unlockId(); + } + } + + } + + /** + * Send log entries to follower, returns true when success, otherwise false and unlock the id. + * + * @param nextSendingIndex next sending index + * @return send result. + */ + private boolean sendEntries(final long nextSendingIndex) { + final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder(); + if (!fillCommonFields(rb, nextSendingIndex - 1, false)) { + // unlock id in installSnapshot + installSnapshot(); + return false; + } + + ByteBufferCollector dataBuf = null; + final int maxEntriesSize = this.raftOptions.getMaxEntriesSize(); + final RecyclableByteBufferList byteBufList = RecyclableByteBufferList.newInstance(); + try { + for (int i = 0; i < maxEntriesSize; i++) { + final RaftOutter.EntryMeta.Builder emb = RaftOutter.EntryMeta.newBuilder(); + if (!prepareEntry(nextSendingIndex, i, emb, byteBufList)) { + break; + } + rb.addEntries(emb.build()); + } + if (rb.getEntriesCount() == 0) { + if (nextSendingIndex < this.options.getLogManager().getFirstLogIndex()) { + installSnapshot(); + return false; + } + // _id is unlock in _wait_more + waitMoreEntries(nextSendingIndex); + return false; + } + if (byteBufList.getCapacity() > 0) { + dataBuf = ByteBufferCollector.allocateByRecyclers(byteBufList.getCapacity()); + for (final ByteBuffer b : byteBufList) { + dataBuf.put(b); + } + final ByteBuffer buf = dataBuf.getBuffer(); + buf.flip(); + rb.setData(ZeroByteStringHelper.wrap(buf)); + } + } finally { + RecycleUtil.recycle(byteBufList); + } + + final AppendEntriesRequest request = rb.build(); + if (LOG.isDebugEnabled()) { + LOG.debug( + "Node {} send AppendEntriesRequest to {} term {} lastCommittedIndex {} prevLogIndex {} prevLogTerm {} logIndex {} count {}", + this.options.getNode().getNodeId(), this.options.getPeerId(), this.options.getTerm(), + request.getCommittedIndex(), request.getPrevLogIndex(), request.getPrevLogTerm(), nextSendingIndex, + request.getEntriesCount()); + } + this.statInfo.runningState = RunningState.APPENDING_ENTRIES; + this.statInfo.firstLogIndex = rb.getPrevLogIndex() + 1; + this.statInfo.lastLogIndex = rb.getPrevLogIndex() + rb.getEntriesCount(); + + final Recyclable recyclable = dataBuf; + final int v = this.version; + final long monotonicSendTimeMs = Utils.monotonicMs(); + final int seq = getAndIncrementReqSeq(); + + this.appendEntriesCounter++; + Future rpcFuture = null; + try { + rpcFuture = this.rpcService.appendEntries(this.options.getPeerId().getEndpoint(), request, -1, + new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + RecycleUtil.recycle(recyclable); // TODO: recycle on send success, not response received. + onRpcReturned(Replicator.this.id, RequestType.AppendEntries, status, request, getResponse(), + seq, v, monotonicSendTimeMs); + } + }); + } catch (final Throwable t) { + RecycleUtil.recycle(recyclable); + ThrowUtil.throwException(t); + } + addInflight(RequestType.AppendEntries, nextSendingIndex, request.getEntriesCount(), request.getData().size(), + seq, rpcFuture); + + return true; + } + + public static void sendHeartbeat(final ThreadId id, final RpcResponseClosure closure) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + RpcUtils.runClosureInThread(closure, new Status(RaftError.EHOSTDOWN, "Peer %s is not connected", id)); + return; + } + //id unlock in send empty entries. + r.sendEmptyEntries(true, closure); + } + + private static void sendHeartbeat(final ThreadId id) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return; + } + // unlock in sendEmptyEntries + r.sendEmptyEntries(true); + } + + private void sendProbeRequest() { + sendEmptyEntries(false); + } + + @SuppressWarnings("SameParameterValue") + private void sendTimeoutNow(final boolean unlockId, final boolean stopAfterFinish) { + sendTimeoutNow(unlockId, stopAfterFinish, -1); + } + + private void sendTimeoutNow(final boolean unlockId, final boolean stopAfterFinish, final int timeoutMs) { + final TimeoutNowRequest.Builder rb = TimeoutNowRequest.newBuilder(); + rb.setTerm(this.options.getTerm()); + rb.setGroupId(this.options.getGroupId()); + rb.setServerId(this.options.getServerId().toString()); + rb.setPeerId(this.options.getPeerId().toString()); + try { + if (!stopAfterFinish) { + // This RPC is issued by transfer_leadership, save this call_id so that + // the RPC can be cancelled by stop. + this.timeoutNowInFly = timeoutNow(rb, false, timeoutMs); + this.timeoutNowIndex = 0; + } else { + timeoutNow(rb, true, timeoutMs); + } + } finally { + if (unlockId) { + unlockId(); + } + } + + } + + private Future timeoutNow(final TimeoutNowRequest.Builder rb, final boolean stopAfterFinish, + final int timeoutMs) { + final TimeoutNowRequest request = rb.build(); + return this.rpcService.timeoutNow(this.options.getPeerId().getEndpoint(), request, timeoutMs, + new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + if (Replicator.this.id != null) { + onTimeoutNowReturned(Replicator.this.id, status, request, getResponse(), stopAfterFinish); + } + } + + }); + } + + @SuppressWarnings("unused") + static void onTimeoutNowReturned(final ThreadId id, final Status status, final TimeoutNowRequest request, + final TimeoutNowResponse response, final boolean stopAfterFinish) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return; + } + + final boolean isLogDebugEnabled = LOG.isDebugEnabled(); + StringBuilder sb = null; + if (isLogDebugEnabled) { + sb = new StringBuilder("Node "). // + append(r.options.getGroupId()).append(":").append(r.options.getServerId()). // + append(" received TimeoutNowResponse from "). // + append(r.options.getPeerId()); + } + if (!status.isOk()) { + if (isLogDebugEnabled) { + sb.append(" fail:").append(status); + LOG.debug(sb.toString()); + } + notifyReplicatorStatusListener(r, ReplicatorEvent.ERROR, status); + if (stopAfterFinish) { + r.notifyOnCaughtUp(RaftError.ESTOP.getNumber(), true); + r.destroy(); + } else { + id.unlock(); + } + return; + } + if (isLogDebugEnabled) { + sb.append(response.getSuccess() ? " success" : " fail"); + LOG.debug(sb.toString()); + } + if (response.getTerm() > r.options.getTerm()) { + final NodeImpl node = r.options.getNode(); + r.notifyOnCaughtUp(RaftError.EPERM.getNumber(), true); + r.destroy(); + node.increaseTermTo(response.getTerm(), new Status(RaftError.EHIGHERTERMRESPONSE, + "Leader receives higher term timeout_now_response from peer:%s", r.options.getPeerId())); + return; + } + if (stopAfterFinish) { + r.notifyOnCaughtUp(RaftError.ESTOP.getNumber(), true); + r.destroy(); + } else { + id.unlock(); + } + + } + + public static boolean stop(final ThreadId id) { + id.setError(RaftError.ESTOP.getNumber()); + return true; + } + + public static boolean join(final ThreadId id) { + id.join(); + return true; + } + + public static long getLastRpcSendTimestamp(final ThreadId id) { + final Replicator r = (Replicator) id.getData(); + if (r == null) { + return 0L; + } + return r.lastRpcSendTimestamp; + } + + public static boolean transferLeadership(final ThreadId id, final long logIndex) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return false; + } + // dummy is unlock in _transfer_leadership + return r.transferLeadership(logIndex); + } + + private boolean transferLeadership(final long logIndex) { + if (this.hasSucceeded && this.nextIndex > logIndex) { + // _id is unlock in _send_timeout_now + sendTimeoutNow(true, false); + return true; + } + // Register log_index so that _on_rpc_return trigger + // _send_timeout_now if _next_index reaches log_index + this.timeoutNowIndex = logIndex; + unlockId(); + return true; + } + + public static boolean stopTransferLeadership(final ThreadId id) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return false; + } + r.timeoutNowIndex = 0; + id.unlock(); + return true; + } + + public static boolean sendTimeoutNowAndStop(final ThreadId id, final int timeoutMs) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return false; + } + // id unlock in sendTimeoutNow + r.sendTimeoutNow(true, true, timeoutMs); + return true; + } + + public static long getNextIndex(final ThreadId id) { + final Replicator r = (Replicator) id.lock(); + if (r == null) { + return 0; + } + long nextIdx = 0; + if (r.hasSucceeded) { + nextIdx = r.nextIndex; + } + id.unlock(); + return nextIdx; + } + + private void unlockId() { + if (this.id == null) { + return; + } + this.id.unlock(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorGroupImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorGroupImpl.java new file mode 100644 index 0000000..41530c5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorGroupImpl.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.closure.CatchUpClosure; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.entity.NodeId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.option.ReplicatorGroupOptions; +import com.alipay.sofa.jraft.option.ReplicatorOptions; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.ThreadId; + +/** + * Replicator group for a raft group. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 1:54:51 PM + */ +public class ReplicatorGroupImpl implements ReplicatorGroup { + + private static final Logger LOG = LoggerFactory + .getLogger(ReplicatorGroupImpl.class); + + // + private final ConcurrentMap replicatorMap = new ConcurrentHashMap<>(); + /** common replicator options */ + private ReplicatorOptions commonOptions; + private int dynamicTimeoutMs = -1; + private int electionTimeoutMs = -1; + private RaftOptions raftOptions; + private final Map failureReplicators = new ConcurrentHashMap<>(); + + @Override + public boolean init(final NodeId nodeId, final ReplicatorGroupOptions opts) { + this.dynamicTimeoutMs = opts.getHeartbeatTimeoutMs(); + this.electionTimeoutMs = opts.getElectionTimeoutMs(); + this.raftOptions = opts.getRaftOptions(); + this.commonOptions = new ReplicatorOptions(); + this.commonOptions.setDynamicHeartBeatTimeoutMs(this.dynamicTimeoutMs); + this.commonOptions.setElectionTimeoutMs(this.electionTimeoutMs); + this.commonOptions.setRaftRpcService(opts.getRaftRpcClientService()); + this.commonOptions.setLogManager(opts.getLogManager()); + this.commonOptions.setBallotBox(opts.getBallotBox()); + this.commonOptions.setNode(opts.getNode()); + this.commonOptions.setTerm(0); + this.commonOptions.setGroupId(nodeId.getGroupId()); + this.commonOptions.setServerId(nodeId.getPeerId()); + this.commonOptions.setSnapshotStorage(opts.getSnapshotStorage()); + this.commonOptions.setTimerManager(opts.getTimerManager()); + return true; + } + + @OnlyForTest + ConcurrentMap getReplicatorMap() { + return this.replicatorMap; + } + + @OnlyForTest + Map getFailureReplicators() { + return this.failureReplicators; + } + + @Override + public void sendHeartbeat(final PeerId peer, final RpcResponseClosure closure) { + final ThreadId rid = this.replicatorMap.get(peer); + if (rid == null) { + if (closure != null) { + closure.run(new Status(RaftError.EHOSTDOWN, "Peer %s is not connected", peer)); + } + return; + } + Replicator.sendHeartbeat(rid, closure); + } + + @Override + public ThreadId getReplicator(final PeerId peer) { + return this.replicatorMap.get(peer); + } + + @Override + public boolean addReplicator(final PeerId peer, final ReplicatorType replicatorType, final boolean sync) { + Requires.requireTrue(this.commonOptions.getTerm() != 0); + this.failureReplicators.remove(peer); + if (this.replicatorMap.containsKey(peer)) { + return true; + } + final ReplicatorOptions opts = this.commonOptions == null ? new ReplicatorOptions() : this.commonOptions.copy(); + opts.setReplicatorType(replicatorType); + opts.setPeerId(peer); + if (!sync) { + final RaftClientService client = opts.getRaftRpcService(); + if (client != null && !client.checkConnection(peer.getEndpoint(), true)) { + LOG.error("Fail to check replicator connection to peer={}, replicatorType={}.", peer, replicatorType); + this.failureReplicators.put(peer, replicatorType); + return false; + } + } + final ThreadId rid = Replicator.start(opts, this.raftOptions); + if (rid == null) { + LOG.error("Fail to start replicator to peer={}, replicatorType={}.", peer, replicatorType); + this.failureReplicators.put(peer, replicatorType); + return false; + } + return this.replicatorMap.put(peer, rid) == null; + } + + @Override + public void clearFailureReplicators() { + this.failureReplicators.clear(); + } + + @Override + public boolean waitCaughtUp(final PeerId peer, final long maxMargin, final long dueTime, final CatchUpClosure done) { + final ThreadId rid = this.replicatorMap.get(peer); + if (rid == null) { + return false; + } + + Replicator.waitForCaughtUp(rid, maxMargin, dueTime, done); + return true; + } + + @Override + public long getLastRpcSendTimestamp(final PeerId peer) { + final ThreadId rid = this.replicatorMap.get(peer); + if (rid == null) { + return 0L; + } + return Replicator.getLastRpcSendTimestamp(rid); + } + + @Override + public boolean stopAll() { + final List rids = new ArrayList<>(this.replicatorMap.values()); + this.replicatorMap.clear(); + this.failureReplicators.clear(); + for (final ThreadId rid : rids) { + Replicator.stop(rid); + } + return true; + } + + @Override + public void checkReplicator(final PeerId peer, final boolean lockNode) { + final ThreadId rid = this.replicatorMap.get(peer); + if (rid == null) { + // Create replicator if it's not found for leader. + final NodeImpl node = this.commonOptions.getNode(); + if (lockNode) { + node.writeLock.lock(); + } + try { + if (node.isLeader()) { + final ReplicatorType rType = this.failureReplicators.get(peer); + if (rType != null && addReplicator(peer, rType, false)) { + this.failureReplicators.remove(peer, rType); + } + } + } finally { + if (lockNode) { + node.writeLock.unlock(); + } + } + } + } + + @Override + public boolean stopReplicator(final PeerId peer) { + LOG.info("Stop replicator to {}.", peer); + this.failureReplicators.remove(peer); + final ThreadId rid = this.replicatorMap.remove(peer); + if (rid == null) { + return false; + } + // Calling ReplicatorId.stop might lead to calling stopReplicator again, + // erase entry first to avoid race condition + return Replicator.stop(rid); + } + + @Override + public boolean resetTerm(final long newTerm) { + if (newTerm <= this.commonOptions.getTerm()) { + return false; + } + this.commonOptions.setTerm(newTerm); + return true; + } + + @Override + public boolean resetHeartbeatInterval(final int newIntervalMs) { + this.dynamicTimeoutMs = newIntervalMs; + return true; + } + + @Override + public boolean resetElectionTimeoutInterval(final int newIntervalMs) { + this.electionTimeoutMs = newIntervalMs; + return true; + } + + @Override + public boolean contains(final PeerId peer) { + return this.replicatorMap.containsKey(peer); + } + + @Override + public boolean transferLeadershipTo(final PeerId peer, final long logIndex) { + final ThreadId rid = this.replicatorMap.get(peer); + return rid != null && Replicator.transferLeadership(rid, logIndex); + } + + @Override + public boolean stopTransferLeadership(final PeerId peer) { + final ThreadId rid = this.replicatorMap.get(peer); + return rid != null && Replicator.stopTransferLeadership(rid); + } + + @Override + public ThreadId stopAllAndFindTheNextCandidate(final ConfigurationEntry conf) { + ThreadId candidate = null; + final PeerId candidateId = findTheNextCandidate(conf); + if (candidateId != null) { + candidate = this.replicatorMap.get(candidateId); + } else { + LOG.info("Fail to find the next candidate."); + } + for (final ThreadId r : this.replicatorMap.values()) { + if (r != candidate) { + Replicator.stop(r); + } + } + this.replicatorMap.clear(); + this.failureReplicators.clear(); + return candidate; + } + + @Override + public PeerId findTheNextCandidate(final ConfigurationEntry conf) { + PeerId peerId = null; + int priority = Integer.MIN_VALUE; + long maxIndex = -1L; + for (final Map.Entry entry : this.replicatorMap.entrySet()) { + if (!conf.contains(entry.getKey())) { + continue; + } + final int nextPriority = entry.getKey().getPriority(); + if (nextPriority == ElectionPriority.NotElected) { + continue; + } + final long nextIndex = Replicator.getNextIndex(entry.getValue()); + if (nextIndex > maxIndex) { + maxIndex = nextIndex; + peerId = entry.getKey(); + priority = peerId.getPriority(); + } else if (nextIndex == maxIndex && nextPriority > priority) { + peerId = entry.getKey(); + priority = peerId.getPriority(); + } + } + + if (maxIndex == -1L) { + return null; + } else { + return peerId; + } + } + + @Override + public List listReplicators() { + return new ArrayList<>(this.replicatorMap.values()); + } + + @Override + public void describe(final Printer out) { + out.print(" replicators: ") // + .println(this.replicatorMap.values()); + out.print(" failureReplicators: ") // + .println(this.failureReplicators); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorType.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorType.java new file mode 100644 index 0000000..c7f4dfb --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/ReplicatorType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +/** + * Replicator role + * @author boyan(boyan@antfin.com) + * + */ +public enum ReplicatorType { + Follower, Learner; + + public final boolean isFollower() { + return this == Follower; + } + + public final boolean isLearner() { + return this == Learner; + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Scheduler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Scheduler.java new file mode 100644 index 0000000..95c70d3 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/Scheduler.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * + * @author jiachun.fjc + */ +public interface Scheduler { + + /** + * Creates and executes a one-shot action that becomes enabled + * after the given delay. + * + * @param command the task to execute + * @param delay the time from now to delay execution + * @param unit the time unit of the delay parameter + * @return a ScheduledFuture representing pending completion of + * the task and whose {@code get()} method will return + * {@code null} upon completion + * scheduled for execution + */ + ScheduledFuture schedule(final Runnable command, final long delay, final TimeUnit unit); + + /** + * Creates and executes a periodic action that becomes enabled first + * after the given initial delay, and subsequently with the given + * period; that is executions will commence after + * {@code initialDelay} then {@code initialDelay+period}, then + * {@code initialDelay + 2 * period}, and so on. + * If any execution of the task + * encounters an exception, subsequent executions are suppressed. + * Otherwise, the task will only terminate via cancellation or + * termination of the executor. If any execution of this task + * takes longer than its period, then subsequent executions + * may start late, but will not concurrently execute. + * + * @param command the task to execute + * @param initialDelay the time to delay first execution + * @param period the period between successive executions + * @param unit the time unit of the initialDelay and period parameters + * @return a ScheduledFuture representing pending completion of + * the task, and whose {@code get()} method will throw an + * exception upon cancellation + */ + ScheduledFuture scheduleAtFixedRate(final Runnable command, final long initialDelay, final long period, + final TimeUnit unit); + + /** + * Creates and executes a periodic action that becomes enabled first + * after the given initial delay, and subsequently with the + * given delay between the termination of one execution and the + * commencement of the next. If any execution of the task + * encounters an exception, subsequent executions are suppressed. + * Otherwise, the task will only terminate via cancellation or + * termination of the executor. + * + * @param command the task to execute + * @param initialDelay the time to delay first execution + * @param delay the delay between the termination of one + * execution and the commencement of the next + * @param unit the time unit of the initialDelay and delay parameters + * @return a ScheduledFuture representing pending completion of + * the task, and whose {@code get()} method will throw an + * exception upon cancellation + */ + ScheduledFuture scheduleWithFixedDelay(final Runnable command, final long initialDelay, final long delay, + final TimeUnit unit); + + void shutdown(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/State.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/State.java new file mode 100644 index 0000000..11ba65e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/State.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +/** + * Node state + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 5:41:54 PM + */ +public enum State { + STATE_LEADER, // It's a leader + STATE_TRANSFERRING, // It's transferring leadership + STATE_CANDIDATE, // It's a candidate + STATE_FOLLOWER, // It's a follower + STATE_ERROR, // It's in error + STATE_UNINITIALIZED, // It's uninitialized + STATE_SHUTTING, // It's shutting down + STATE_SHUTDOWN, // It's shutdown already + STATE_END; // State end + + public boolean isActive() { + return this.ordinal() < STATE_ERROR.ordinal(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/StateMachineAdapter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/StateMachineAdapter.java new file mode 100644 index 0000000..08bac3b --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/StateMachineAdapter.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.LeaderChangeContext; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; + +/** + * State machine adapter that implements all methods with default behavior + * except {@link #onApply(com.alipay.sofa.jraft.Iterator)}. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-30 10:32:27 AM + */ +public abstract class StateMachineAdapter implements StateMachine { + + private static final Logger LOG = LoggerFactory.getLogger(StateMachineAdapter.class); + + @Override + public void onShutdown() { + LOG.info("onShutdown."); + } + + @Override + public void onSnapshotSave(final SnapshotWriter writer, final Closure done) { + error("onSnapshotSave"); + runClosure(done, "onSnapshotSave"); + } + + @Override + public boolean onSnapshotLoad(final SnapshotReader reader) { + error("onSnapshotLoad", "while a snapshot is saved in " + reader.getPath()); + return false; + } + + @Override + public void onLeaderStart(final long term) { + LOG.info("onLeaderStart: term={}.", term); + } + + @Override + public void onLeaderStop(final Status status) { + LOG.info("onLeaderStop: status={}.", status); + } + + @Override + public void onError(final RaftException e) { + LOG.error( + "Encountered an error={} on StateMachine {}, it's highly recommended to implement this method as raft stops working since some error occurs, you should figure out the cause and repair or remove this node.", + e.getStatus(), getClassName(), e); + } + + @Override + public void onConfigurationCommitted(final Configuration conf) { + LOG.info("onConfigurationCommitted: {}.", conf); + } + + @Override + public void onStopFollowing(final LeaderChangeContext ctx) { + LOG.info("onStopFollowing: {}.", ctx); + } + + @Override + public void onStartFollowing(final LeaderChangeContext ctx) { + LOG.info("onStartFollowing: {}.", ctx); + } + + @SuppressWarnings("SameParameterValue") + private void runClosure(final Closure done, final String methodName) { + done.run(new Status(-1, "%s doesn't implement %s", getClassName(), methodName)); + } + + private String getClassName() { + return getClass().getName(); + } + + @SuppressWarnings("SameParameterValue") + private void error(final String methodName) { + error(methodName, ""); + } + + private void error(final String methodName, final String msg) { + LOG.error("{} doesn't implement {} {}.", getClassName(), methodName, msg); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/core/TimerManager.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/TimerManager.java new file mode 100644 index 0000000..1939de6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/core/TimerManager.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.core; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; + +/** + * The global timer manager. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-30 3:24:34 PM + */ +public class TimerManager implements Scheduler { + + private final ScheduledExecutorService executor; + + public TimerManager(int workerNum) { + this(workerNum, "JRaft-Node-ScheduleThreadPool"); + } + + public TimerManager(int workerNum, String name) { + this.executor = ThreadPoolUtil.newScheduledBuilder() // + .poolName(name) // + .coreThreads(workerNum) // + .enableMetric(true) // + .threadFactory(new NamedThreadFactory(name, true)) // + .build(); + } + + @Override + public ScheduledFuture schedule(final Runnable command, final long delay, final TimeUnit unit) { + return this.executor.schedule(command, delay, unit); + } + + @Override + public ScheduledFuture scheduleAtFixedRate(final Runnable command, final long initialDelay, final long period, + final TimeUnit unit) { + return this.executor.scheduleAtFixedRate(command, initialDelay, period, unit); + } + + @Override + public ScheduledFuture scheduleWithFixedDelay(final Runnable command, final long initialDelay, final long delay, + final TimeUnit unit) { + return this.executor.scheduleWithFixedDelay(command, initialDelay, delay, unit); + } + + @Override + public void shutdown() { + this.executor.shutdownNow(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Ballot.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Ballot.java new file mode 100644 index 0000000..5f9f1eb --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Ballot.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.util.ArrayList; +import java.util.List; + +import com.alipay.sofa.jraft.conf.Configuration; + +/** + * A ballot to vote. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-15 2:29:11 PM + */ +public class Ballot { + + public static final class PosHint { + int pos0 = -1; // position in current peers + int pos1 = -1; // position in old peers + } + + public static class UnfoundPeerId { + PeerId peerId; + boolean found; + int index; + + public UnfoundPeerId(PeerId peerId, int index, boolean found) { + super(); + this.peerId = peerId; + this.index = index; + this.found = found; + } + } + + private final List peers = new ArrayList<>(); + private int quorum; + private final List oldPeers = new ArrayList<>(); + private int oldQuorum; + + /** + * Init the ballot with current conf and old conf. + * + * @param conf current configuration + * @param oldConf old configuration + * @return true if init success + */ + public boolean init(final Configuration conf, final Configuration oldConf) { + this.peers.clear(); + this.oldPeers.clear(); + this.quorum = this.oldQuorum = 0; + int index = 0; + if (conf != null) { + for (final PeerId peer : conf) { + this.peers.add(new UnfoundPeerId(peer, index++, false)); + } + } + + this.quorum = this.peers.size() / 2 + 1; + if (oldConf == null) { + return true; + } + index = 0; + for (final PeerId peer : oldConf) { + this.oldPeers.add(new UnfoundPeerId(peer, index++, false)); + } + + this.oldQuorum = this.oldPeers.size() / 2 + 1; + return true; + } + + private UnfoundPeerId findPeer(final PeerId peerId, final List peers, final int posHint) { + if (posHint < 0 || posHint >= peers.size() || !peers.get(posHint).peerId.equals(peerId)) { + for (final UnfoundPeerId ufp : peers) { + if (ufp.peerId.equals(peerId)) { + return ufp; + } + } + return null; + } + + return peers.get(posHint); + } + + public PosHint grant(final PeerId peerId, final PosHint hint) { + UnfoundPeerId peer = findPeer(peerId, this.peers, hint.pos0); + if (peer != null) { + if (!peer.found) { + peer.found = true; + this.quorum--; + } + hint.pos0 = peer.index; + } else { + hint.pos0 = -1; + } + if (this.oldPeers.isEmpty()) { + hint.pos1 = -1; + return hint; + } + peer = findPeer(peerId, this.oldPeers, hint.pos1); + if (peer != null) { + if (!peer.found) { + peer.found = true; + this.oldQuorum--; + } + hint.pos1 = peer.index; + } else { + hint.pos1 = -1; + } + + return hint; + } + + public void grant(final PeerId peerId) { + grant(peerId, new PosHint()); + } + + /** + * Returns true when the ballot is granted. + * + * @return true if the ballot is granted + */ + public boolean isGranted() { + return this.quorum <= 0 && this.oldQuorum <= 0; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Checksum.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Checksum.java new file mode 100644 index 0000000..58444ff --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Checksum.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.util.Collection; + +/** + * Checksum for entity. + * + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + */ +public interface Checksum { + + /** + * Calculate a checksum value for this entity. + * @return checksum value + */ + long checksum(); + + /** + * Returns the checksum value of two long values. + * + * @param v1 first long value + * @param v2 second long value + * @return checksum value + */ + default long checksum(final long v1, final long v2) { + return v1 ^ v2; + } + + /** + * Returns the checksum value of act on factors. + * + * @param factors checksum collection + * @param v origin checksum + * @return checksum value + */ + default long checksum(final Collection factors, long v) { + if (factors != null && !factors.isEmpty()) { + for (final Checksum factor : factors) { + v = checksum(v, factor.checksum()); + } + } + return v; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/EnumOutter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/EnumOutter.java new file mode 100644 index 0000000..4223b5c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/EnumOutter.java @@ -0,0 +1,290 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: enum.proto + +package com.alipay.sofa.jraft.entity; + +public final class EnumOutter { + private EnumOutter() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + /** + * Protobuf enum {@code jraft.EntryType} + */ + public enum EntryType implements com.google.protobuf.ProtocolMessageEnum { + /** + * ENTRY_TYPE_UNKNOWN = 0; + */ + ENTRY_TYPE_UNKNOWN(0), + /** + * ENTRY_TYPE_NO_OP = 1; + */ + ENTRY_TYPE_NO_OP(1), + /** + * ENTRY_TYPE_DATA = 2; + */ + ENTRY_TYPE_DATA(2), + /** + * ENTRY_TYPE_CONFIGURATION = 3; + */ + ENTRY_TYPE_CONFIGURATION(3), ; + + /** + * ENTRY_TYPE_UNKNOWN = 0; + */ + public static final int ENTRY_TYPE_UNKNOWN_VALUE = 0; + /** + * ENTRY_TYPE_NO_OP = 1; + */ + public static final int ENTRY_TYPE_NO_OP_VALUE = 1; + /** + * ENTRY_TYPE_DATA = 2; + */ + public static final int ENTRY_TYPE_DATA_VALUE = 2; + /** + * ENTRY_TYPE_CONFIGURATION = 3; + */ + public static final int ENTRY_TYPE_CONFIGURATION_VALUE = 3; + + public final int getNumber() { + return value; + } + + /** + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static EntryType valueOf(int value) { + return forNumber(value); + } + + public static EntryType forNumber(int value) { + switch (value) { + case 0: + return ENTRY_TYPE_UNKNOWN; + case 1: + return ENTRY_TYPE_NO_OP; + case 2: + return ENTRY_TYPE_DATA; + case 3: + return ENTRY_TYPE_CONFIGURATION; + default: + return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap internalGetValueMap() { + return internalValueMap; + } + + private static final com.google.protobuf.Internal.EnumLiteMap internalValueMap = new com.google.protobuf.Internal.EnumLiteMap() { + public EntryType findValueByNumber(int number) { + return EntryType + .forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() { + return getDescriptor().getValues().get(ordinal()); + } + + public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() { + return getDescriptor(); + } + + public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.EnumOutter.getDescriptor().getEnumTypes().get(0); + } + + private static final EntryType[] VALUES = values(); + + public static EntryType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException("EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private EntryType(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:jraft.EntryType) + } + + /** + * Protobuf enum {@code jraft.ErrorType} + */ + public enum ErrorType implements com.google.protobuf.ProtocolMessageEnum { + /** + * ERROR_TYPE_NONE = 0; + */ + ERROR_TYPE_NONE(0), + /** + * ERROR_TYPE_LOG = 1; + */ + ERROR_TYPE_LOG(1), + /** + * ERROR_TYPE_STABLE = 2; + */ + ERROR_TYPE_STABLE(2), + /** + * ERROR_TYPE_SNAPSHOT = 3; + */ + ERROR_TYPE_SNAPSHOT(3), + /** + * ERROR_TYPE_STATE_MACHINE = 4; + */ + ERROR_TYPE_STATE_MACHINE(4), + /** + * ERROR_TYPE_META = 5; + */ + ERROR_TYPE_META(5), ; + + /** + * ERROR_TYPE_NONE = 0; + */ + public static final int ERROR_TYPE_NONE_VALUE = 0; + /** + * ERROR_TYPE_LOG = 1; + */ + public static final int ERROR_TYPE_LOG_VALUE = 1; + /** + * ERROR_TYPE_STABLE = 2; + */ + public static final int ERROR_TYPE_STABLE_VALUE = 2; + /** + * ERROR_TYPE_SNAPSHOT = 3; + */ + public static final int ERROR_TYPE_SNAPSHOT_VALUE = 3; + /** + * ERROR_TYPE_STATE_MACHINE = 4; + */ + public static final int ERROR_TYPE_STATE_MACHINE_VALUE = 4; + /** + * ERROR_TYPE_META = 5; + */ + public static final int ERROR_TYPE_META_VALUE = 5; + + public final int getNumber() { + return value; + } + + /** + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static ErrorType valueOf(int value) { + return forNumber(value); + } + + public static ErrorType forNumber(int value) { + switch (value) { + case 0: + return ERROR_TYPE_NONE; + case 1: + return ERROR_TYPE_LOG; + case 2: + return ERROR_TYPE_STABLE; + case 3: + return ERROR_TYPE_SNAPSHOT; + case 4: + return ERROR_TYPE_STATE_MACHINE; + case 5: + return ERROR_TYPE_META; + default: + return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap internalGetValueMap() { + return internalValueMap; + } + + private static final com.google.protobuf.Internal.EnumLiteMap internalValueMap = new com.google.protobuf.Internal.EnumLiteMap() { + public ErrorType findValueByNumber(int number) { + return ErrorType + .forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() { + return getDescriptor().getValues().get(ordinal()); + } + + public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() { + return getDescriptor(); + } + + public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.EnumOutter.getDescriptor().getEnumTypes().get(1); + } + + private static final ErrorType[] VALUES = values(); + + public static ErrorType valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException("EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private ErrorType(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:jraft.ErrorType) + } + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\nenum.proto\022\005jraft*l\n\tEntryType\022\026\n\022ENTR" + + "Y_TYPE_UNKNOWN\020\000\022\024\n\020ENTRY_TYPE_NO_OP\020\001\022\023" + + "\n\017ENTRY_TYPE_DATA\020\002\022\034\n\030ENTRY_TYPE_CONFIG" + + "URATION\020\003*\227\001\n\tErrorType\022\023\n\017ERROR_TYPE_NO" + + "NE\020\000\022\022\n\016ERROR_TYPE_LOG\020\001\022\025\n\021ERROR_TYPE_S" + + "TABLE\020\002\022\027\n\023ERROR_TYPE_SNAPSHOT\020\003\022\034\n\030ERRO" + + "R_TYPE_STATE_MACHINE\020\004\022\023\n\017ERROR_TYPE_MET" + + "A\020\005B*\n\034com.alipay.sofa.jraft.entityB\nEnu" + "mOutter" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] {}, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LeaderChangeContext.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LeaderChangeContext.java new file mode 100644 index 0000000..ed035a8 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LeaderChangeContext.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import com.alipay.sofa.jraft.Status; + +/** + * The leader change context, contains: + *
    + *
  • leaderId: the leader peer id.
    • + *
  • term: the leader term.
    • + *
  • Status: context status.
    • + *
+ * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-13 3:23:48 PM + */ +public class LeaderChangeContext { + + private PeerId leaderId; + private long term; + private Status status; + + public LeaderChangeContext(PeerId leaderId, long term, Status status) { + super(); + this.leaderId = leaderId; + this.term = term; + this.status = status; + } + + public PeerId getLeaderId() { + return this.leaderId; + } + + public void setLeaderId(PeerId leaderId) { + this.leaderId = leaderId; + } + + public long getTerm() { + return this.term; + } + + public void setTerm(long term) { + this.term = term; + } + + public Status getStatus() { + return this.status; + } + + public void setStatus(Status status) { + this.status = status; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (this.leaderId == null ? 0 : this.leaderId.hashCode()); + result = prime * result + (this.status == null ? 0 : this.status.hashCode()); + result = prime * result + (int) (this.term ^ this.term >>> 32); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + LeaderChangeContext other = (LeaderChangeContext) obj; + if (this.leaderId == null) { + if (other.leaderId != null) { + return false; + } + } else if (!this.leaderId.equals(other.leaderId)) { + return false; + } + if (this.status == null) { + if (other.status != null) { + return false; + } + } else if (!this.status.equals(other.status)) { + return false; + } + return this.term == other.term; + } + + @Override + public String toString() { + return "LeaderChangeContext [leaderId=" + this.leaderId + ", term=" + this.term + ", status=" + this.status + + "]"; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java new file mode 100644 index 0000000..4a1aa6d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalFileMetaOutter.java @@ -0,0 +1,911 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: local_file_meta.proto + +package com.alipay.sofa.jraft.entity; + +public final class LocalFileMetaOutter { + private LocalFileMetaOutter() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + /** + * Protobuf enum {@code jraft.FileSource} + */ + public enum FileSource implements com.google.protobuf.ProtocolMessageEnum { + /** + * FILE_SOURCE_LOCAL = 0; + */ + FILE_SOURCE_LOCAL(0), + /** + * FILE_SOURCE_REFERENCE = 1; + */ + FILE_SOURCE_REFERENCE(1), ; + + /** + * FILE_SOURCE_LOCAL = 0; + */ + public static final int FILE_SOURCE_LOCAL_VALUE = 0; + /** + * FILE_SOURCE_REFERENCE = 1; + */ + public static final int FILE_SOURCE_REFERENCE_VALUE = 1; + + public final int getNumber() { + return value; + } + + /** + * @deprecated Use {@link #forNumber(int)} instead. + */ + @java.lang.Deprecated + public static FileSource valueOf(int value) { + return forNumber(value); + } + + public static FileSource forNumber(int value) { + switch (value) { + case 0: + return FILE_SOURCE_LOCAL; + case 1: + return FILE_SOURCE_REFERENCE; + default: + return null; + } + } + + public static com.google.protobuf.Internal.EnumLiteMap internalGetValueMap() { + return internalValueMap; + } + + private static final com.google.protobuf.Internal.EnumLiteMap internalValueMap = new com.google.protobuf.Internal.EnumLiteMap() { + public FileSource findValueByNumber(int number) { + return FileSource + .forNumber(number); + } + }; + + public final com.google.protobuf.Descriptors.EnumValueDescriptor getValueDescriptor() { + return getDescriptor().getValues().get(ordinal()); + } + + public final com.google.protobuf.Descriptors.EnumDescriptor getDescriptorForType() { + return getDescriptor(); + } + + public static final com.google.protobuf.Descriptors.EnumDescriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.getDescriptor().getEnumTypes().get(0); + } + + private static final FileSource[] VALUES = values(); + + public static FileSource valueOf(com.google.protobuf.Descriptors.EnumValueDescriptor desc) { + if (desc.getType() != getDescriptor()) { + throw new java.lang.IllegalArgumentException("EnumValueDescriptor is not for this type."); + } + return VALUES[desc.getIndex()]; + } + + private final int value; + + private FileSource(int value) { + this.value = value; + } + + // @@protoc_insertion_point(enum_scope:jraft.FileSource) + } + + public interface LocalFileMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.LocalFileMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * optional bytes user_meta = 1; + */ + boolean hasUserMeta(); + + /** + * optional bytes user_meta = 1; + */ + com.google.protobuf.ByteString getUserMeta(); + + /** + * optional .jraft.FileSource source = 2; + */ + boolean hasSource(); + + /** + * optional .jraft.FileSource source = 2; + */ + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource getSource(); + + /** + * optional string checksum = 3; + */ + boolean hasChecksum(); + + /** + * optional string checksum = 3; + */ + java.lang.String getChecksum(); + + /** + * optional string checksum = 3; + */ + com.google.protobuf.ByteString getChecksumBytes(); + } + + /** + * Protobuf type {@code jraft.LocalFileMeta} + */ + public static final class LocalFileMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.LocalFileMeta) + LocalFileMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use LocalFileMeta.newBuilder() to construct. + private LocalFileMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private LocalFileMeta() { + userMeta_ = com.google.protobuf.ByteString.EMPTY; + source_ = 0; + checksum_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private LocalFileMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + bitField0_ |= 0x00000001; + userMeta_ = input.readBytes(); + break; + } + case 16: { + int rawValue = input.readEnum(); + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource value = com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource + .valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + bitField0_ |= 0x00000002; + source_ = rawValue; + } + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + checksum_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.internal_static_jraft_LocalFileMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.internal_static_jraft_LocalFileMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.class, + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder.class); + } + + private int bitField0_; + public static final int USER_META_FIELD_NUMBER = 1; + private com.google.protobuf.ByteString userMeta_; + + /** + * optional bytes user_meta = 1; + */ + public boolean hasUserMeta() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional bytes user_meta = 1; + */ + public com.google.protobuf.ByteString getUserMeta() { + return userMeta_; + } + + public static final int SOURCE_FIELD_NUMBER = 2; + private int source_; + + /** + * optional .jraft.FileSource source = 2; + */ + public boolean hasSource() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.FileSource source = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource getSource() { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource result = com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource + .valueOf(source_); + return result == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL + : result; + } + + public static final int CHECKSUM_FIELD_NUMBER = 3; + private volatile java.lang.Object checksum_; + + /** + * optional string checksum = 3; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional string checksum = 3; + */ + public java.lang.String getChecksum() { + java.lang.Object ref = checksum_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + checksum_ = s; + } + return s; + } + } + + /** + * optional string checksum = 3; + */ + public com.google.protobuf.ByteString getChecksumBytes() { + java.lang.Object ref = checksum_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + checksum_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBytes(1, userMeta_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeEnum(2, source_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, checksum_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeBytesSize(1, userMeta_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeEnumSize(2, source_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, checksum_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta other = (com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) obj; + + boolean result = true; + result = result && (hasUserMeta() == other.hasUserMeta()); + if (hasUserMeta()) { + result = result && getUserMeta().equals(other.getUserMeta()); + } + result = result && (hasSource() == other.hasSource()); + if (hasSource()) { + result = result && source_ == other.source_; + } + result = result && (hasChecksum() == other.hasChecksum()); + if (hasChecksum()) { + result = result && getChecksum().equals(other.getChecksum()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasUserMeta()) { + hash = (37 * hash) + USER_META_FIELD_NUMBER; + hash = (53 * hash) + getUserMeta().hashCode(); + } + if (hasSource()) { + hash = (37 * hash) + SOURCE_FIELD_NUMBER; + hash = (53 * hash) + source_; + } + if (hasChecksum()) { + hash = (37 * hash) + CHECKSUM_FIELD_NUMBER; + hash = (53 * hash) + getChecksum().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.LocalFileMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.LocalFileMeta) + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.internal_static_jraft_LocalFileMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.internal_static_jraft_LocalFileMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.class, + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + userMeta_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + source_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + checksum_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.internal_static_jraft_LocalFileMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta build() { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta buildPartial() { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta result = new com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.userMeta_ = userMeta_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.source_ = source_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.checksum_ = checksum_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta other) { + if (other == com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.getDefaultInstance()) + return this; + if (other.hasUserMeta()) { + setUserMeta(other.getUserMeta()); + } + if (other.hasSource()) { + setSource(other.getSource()); + } + if (other.hasChecksum()) { + bitField0_ |= 0x00000004; + checksum_ = other.checksum_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.ByteString userMeta_ = com.google.protobuf.ByteString.EMPTY; + + /** + * optional bytes user_meta = 1; + */ + public boolean hasUserMeta() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional bytes user_meta = 1; + */ + public com.google.protobuf.ByteString getUserMeta() { + return userMeta_; + } + + /** + * optional bytes user_meta = 1; + */ + public Builder setUserMeta(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + userMeta_ = value; + onChanged(); + return this; + } + + /** + * optional bytes user_meta = 1; + */ + public Builder clearUserMeta() { + bitField0_ = (bitField0_ & ~0x00000001); + userMeta_ = getDefaultInstance().getUserMeta(); + onChanged(); + return this; + } + + private int source_ = 0; + + /** + * optional .jraft.FileSource source = 2; + */ + public boolean hasSource() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.FileSource source = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource getSource() { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource result = com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource + .valueOf(source_); + return result == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL + : result; + } + + /** + * optional .jraft.FileSource source = 2; + */ + public Builder setSource(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + source_ = value.getNumber(); + onChanged(); + return this; + } + + /** + * optional .jraft.FileSource source = 2; + */ + public Builder clearSource() { + bitField0_ = (bitField0_ & ~0x00000002); + source_ = 0; + onChanged(); + return this; + } + + private java.lang.Object checksum_ = ""; + + /** + * optional string checksum = 3; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional string checksum = 3; + */ + public java.lang.String getChecksum() { + java.lang.Object ref = checksum_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + checksum_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string checksum = 3; + */ + public com.google.protobuf.ByteString getChecksumBytes() { + java.lang.Object ref = checksum_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + checksum_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string checksum = 3; + */ + public Builder setChecksum(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + checksum_ = value; + onChanged(); + return this; + } + + /** + * optional string checksum = 3; + */ + public Builder clearChecksum() { + bitField0_ = (bitField0_ & ~0x00000004); + checksum_ = getDefaultInstance().getChecksum(); + onChanged(); + return this; + } + + /** + * optional string checksum = 3; + */ + public Builder setChecksumBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + checksum_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.LocalFileMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.LocalFileMeta) + private static final com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta(); + } + + public static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public LocalFileMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new LocalFileMeta(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_LocalFileMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_LocalFileMeta_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\025local_file_meta.proto\022\005jraft\"W\n\rLocalF" + + "ileMeta\022\021\n\tuser_meta\030\001 \001(\014\022!\n\006source\030\002 \001" + + "(\0162\021.jraft.FileSource\022\020\n\010checksum\030\003 \001(\t*" + + ">\n\nFileSource\022\025\n\021FILE_SOURCE_LOCAL\020\000\022\031\n\025" + + "FILE_SOURCE_REFERENCE\020\001B3\n\034com.alipay.so" + + "fa.jraft.entityB\023LocalFileMetaOutter" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] {}, assigner); + internal_static_jraft_LocalFileMeta_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_LocalFileMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_LocalFileMeta_descriptor, + new java.lang.String[] { "UserMeta", "Source", "Checksum", }); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalStorageOutter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalStorageOutter.java new file mode 100644 index 0000000..8560c06 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LocalStorageOutter.java @@ -0,0 +1,3805 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: local_storage.proto + +package com.alipay.sofa.jraft.entity; + +public final class LocalStorageOutter { + private LocalStorageOutter() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public interface ConfigurationPBMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ConfigurationPBMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string peers = 1; + */ + java.util.List getPeersList(); + + /** + * repeated string peers = 1; + */ + int getPeersCount(); + + /** + * repeated string peers = 1; + */ + java.lang.String getPeers(int index); + + /** + * repeated string peers = 1; + */ + com.google.protobuf.ByteString getPeersBytes(int index); + + /** + * repeated string old_peers = 2; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 2; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 2; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 2; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + } + + /** + * Protobuf type {@code jraft.ConfigurationPBMeta} + */ + public static final class ConfigurationPBMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ConfigurationPBMeta) + ConfigurationPBMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ConfigurationPBMeta.newBuilder() to construct. + private ConfigurationPBMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ConfigurationPBMeta() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ConfigurationPBMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + peers_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + oldPeers_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = peers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_ConfigurationPBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_ConfigurationPBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.Builder.class); + } + + public static final int PEERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList peers_; + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_; + } + + /** + * repeated string peers = 1; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 1; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + public static final int OLD_PEERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 2; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 2; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 2; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < peers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, peers_.getRaw(i)); + } + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, oldPeers_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < peers_.size(); i++) { + dataSize += computeStringSizeNoTag(peers_.getRaw(i)); + } + size += dataSize; + size += 1 * getPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta other = (com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta) obj; + + boolean result = true; + result = result && getPeersList().equals(other.getPeersList()); + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getPeersCount() > 0) { + hash = (37 * hash) + PEERS_FIELD_NUMBER; + hash = (53 * hash) + getPeersList().hashCode(); + } + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ConfigurationPBMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ConfigurationPBMeta) + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_ConfigurationPBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_ConfigurationPBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_ConfigurationPBMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta build() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta buildPartial() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta result = new com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta( + this); + int from_bitField0_ = bitField0_; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = peers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.peers_ = peers_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.oldPeers_ = oldPeers_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta other) { + if (other == com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta.getDefaultInstance()) + return this; + if (!other.peers_.isEmpty()) { + if (peers_.isEmpty()) { + peers_ = other.peers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensurePeersIsMutable(); + peers_.addAll(other.peers_); + } + onChanged(); + } + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensurePeersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = new com.google.protobuf.LazyStringArrayList(peers_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_.getUnmodifiableView(); + } + + /** + * repeated string peers = 1; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 1; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + /** + * repeated string peers = 1; + */ + public Builder setPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addAllPeers(java.lang.Iterable values) { + ensurePeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, peers_); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder clearPeers() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string old_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 2; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 2; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 2; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 2; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 2; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 2; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 2; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 2; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ConfigurationPBMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.ConfigurationPBMeta) + private static final com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta(); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ConfigurationPBMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ConfigurationPBMeta( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.ConfigurationPBMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface LogPBMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.LogPBMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 first_log_index = 1; + */ + boolean hasFirstLogIndex(); + + /** + * required int64 first_log_index = 1; + */ + long getFirstLogIndex(); + } + + /** + * Protobuf type {@code jraft.LogPBMeta} + */ + public static final class LogPBMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.LogPBMeta) + LogPBMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use LogPBMeta.newBuilder() to construct. + private LogPBMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private LogPBMeta() { + firstLogIndex_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private LogPBMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + firstLogIndex_ = input.readInt64(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LogPBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LogPBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.Builder.class); + } + + private int bitField0_; + public static final int FIRST_LOG_INDEX_FIELD_NUMBER = 1; + private long firstLogIndex_; + + /** + * required int64 first_log_index = 1; + */ + public boolean hasFirstLogIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 first_log_index = 1; + */ + public long getFirstLogIndex() { + return firstLogIndex_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasFirstLogIndex()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, firstLogIndex_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, firstLogIndex_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta other = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta) obj; + + boolean result = true; + result = result && (hasFirstLogIndex() == other.hasFirstLogIndex()); + if (hasFirstLogIndex()) { + result = result && (getFirstLogIndex() == other.getFirstLogIndex()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasFirstLogIndex()) { + hash = (37 * hash) + FIRST_LOG_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getFirstLogIndex()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.LogPBMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.LogPBMeta) + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LogPBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LogPBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + firstLogIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LogPBMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta build() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta buildPartial() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta result = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.firstLogIndex_ = firstLogIndex_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta other) { + if (other == com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta.getDefaultInstance()) + return this; + if (other.hasFirstLogIndex()) { + setFirstLogIndex(other.getFirstLogIndex()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasFirstLogIndex()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long firstLogIndex_; + + /** + * required int64 first_log_index = 1; + */ + public boolean hasFirstLogIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 first_log_index = 1; + */ + public long getFirstLogIndex() { + return firstLogIndex_; + } + + /** + * required int64 first_log_index = 1; + */ + public Builder setFirstLogIndex(long value) { + bitField0_ |= 0x00000001; + firstLogIndex_ = value; + onChanged(); + return this; + } + + /** + * required int64 first_log_index = 1; + */ + public Builder clearFirstLogIndex() { + bitField0_ = (bitField0_ & ~0x00000001); + firstLogIndex_ = 0L; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.LogPBMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.LogPBMeta) + private static final com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta(); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public LogPBMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new LogPBMeta(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LogPBMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface StablePBMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.StablePBMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required string votedfor = 2; + */ + boolean hasVotedfor(); + + /** + * required string votedfor = 2; + */ + java.lang.String getVotedfor(); + + /** + * required string votedfor = 2; + */ + com.google.protobuf.ByteString getVotedforBytes(); + } + + /** + * Protobuf type {@code jraft.StablePBMeta} + */ + public static final class StablePBMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.StablePBMeta) + StablePBMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use StablePBMeta.newBuilder() to construct. + private StablePBMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private StablePBMeta() { + term_ = 0L; + votedfor_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private StablePBMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + votedfor_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_StablePBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_StablePBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int VOTEDFOR_FIELD_NUMBER = 2; + private volatile java.lang.Object votedfor_; + + /** + * required string votedfor = 2; + */ + public boolean hasVotedfor() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string votedfor = 2; + */ + public java.lang.String getVotedfor() { + java.lang.Object ref = votedfor_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + votedfor_ = s; + } + return s; + } + } + + /** + * required string votedfor = 2; + */ + public com.google.protobuf.ByteString getVotedforBytes() { + java.lang.Object ref = votedfor_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + votedfor_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasVotedfor()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, votedfor_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, votedfor_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta other = (com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasVotedfor() == other.hasVotedfor()); + if (hasVotedfor()) { + result = result && getVotedfor().equals(other.getVotedfor()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasVotedfor()) { + hash = (37 * hash) + VOTEDFOR_FIELD_NUMBER; + hash = (53 * hash) + getVotedfor().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.StablePBMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.StablePBMeta) + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_StablePBMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_StablePBMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + votedfor_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_StablePBMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta build() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta buildPartial() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta result = new com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.votedfor_ = votedfor_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta other) { + if (other == com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasVotedfor()) { + bitField0_ |= 0x00000002; + votedfor_ = other.votedfor_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasVotedfor()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private java.lang.Object votedfor_ = ""; + + /** + * required string votedfor = 2; + */ + public boolean hasVotedfor() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string votedfor = 2; + */ + public java.lang.String getVotedfor() { + java.lang.Object ref = votedfor_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + votedfor_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string votedfor = 2; + */ + public com.google.protobuf.ByteString getVotedforBytes() { + java.lang.Object ref = votedfor_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + votedfor_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string votedfor = 2; + */ + public Builder setVotedfor(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + votedfor_ = value; + onChanged(); + return this; + } + + /** + * required string votedfor = 2; + */ + public Builder clearVotedfor() { + bitField0_ = (bitField0_ & ~0x00000002); + votedfor_ = getDefaultInstance().getVotedfor(); + onChanged(); + return this; + } + + /** + * required string votedfor = 2; + */ + public Builder setVotedforBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + votedfor_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.StablePBMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.StablePBMeta) + private static final com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta(); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public StablePBMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new StablePBMeta(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface LocalSnapshotPbMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.LocalSnapshotPbMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + boolean hasMeta(); + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta(); + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder(); + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + java.util.List getFilesList(); + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getFiles(int index); + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + int getFilesCount(); + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + java.util.List getFilesOrBuilderList(); + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.FileOrBuilder getFilesOrBuilder(int index); + } + + /** + * Protobuf type {@code jraft.LocalSnapshotPbMeta} + */ + public static final class LocalSnapshotPbMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.LocalSnapshotPbMeta) + LocalSnapshotPbMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use LocalSnapshotPbMeta.newBuilder() to construct. + private LocalSnapshotPbMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private LocalSnapshotPbMeta() { + files_ = java.util.Collections.emptyList(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private LocalSnapshotPbMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = meta_.toBuilder(); + } + meta_ = input.readMessage(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.PARSER, + extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(meta_); + meta_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + case 18: { + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + files_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000002; + } + files_.add(input.readMessage( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.PARSER, + extensionRegistry)); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + files_ = java.util.Collections.unmodifiableList(files_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.Builder.class); + } + + public interface FileOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.LocalSnapshotPbMeta.File) + com.google.protobuf.MessageOrBuilder { + + /** + * required string name = 1; + */ + boolean hasName(); + + /** + * required string name = 1; + */ + java.lang.String getName(); + + /** + * required string name = 1; + */ + com.google.protobuf.ByteString getNameBytes(); + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + boolean hasMeta(); + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getMeta(); + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMetaOrBuilder getMetaOrBuilder(); + } + + /** + * Protobuf type {@code jraft.LocalSnapshotPbMeta.File} + */ + public static final class File extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.LocalSnapshotPbMeta.File) + FileOrBuilder { + private static final long serialVersionUID = 0L; + + // Use File.newBuilder() to construct. + private File(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private File() { + name_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private File(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + name_ = bs; + break; + } + case 18: { + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder subBuilder = null; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + subBuilder = meta_.toBuilder(); + } + meta_ = input.readMessage( + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.PARSER, + extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(meta_); + meta_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000002; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_File_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_File_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder.class); + } + + private int bitField0_; + public static final int NAME_FIELD_NUMBER = 1; + private volatile java.lang.Object name_; + + /** + * required string name = 1; + */ + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string name = 1; + */ + public java.lang.String getName() { + java.lang.Object ref = name_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + name_ = s; + } + return s; + } + } + + /** + * required string name = 1; + */ + public com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int META_FIELD_NUMBER = 2; + private com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta meta_; + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getMeta() { + return meta_ == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta + .getDefaultInstance() : meta_; + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMetaOrBuilder getMetaOrBuilder() { + return meta_ == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta + .getDefaultInstance() : meta_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasName()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, name_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeMessage(2, getMeta()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, name_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(2, getMeta()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File other = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File) obj; + + boolean result = true; + result = result && (hasName() == other.hasName()); + if (hasName()) { + result = result && getName().equals(other.getName()); + } + result = result && (hasMeta() == other.hasMeta()); + if (hasMeta()) { + result = result && getMeta().equals(other.getMeta()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasName()) { + hash = (37 * hash) + NAME_FIELD_NUMBER; + hash = (53 * hash) + getName().hashCode(); + } + if (hasMeta()) { + hash = (37 * hash) + META_FIELD_NUMBER; + hash = (53 * hash) + getMeta().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.LocalSnapshotPbMeta.File} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.LocalSnapshotPbMeta.File) + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.FileOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_File_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_File_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getMetaFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + name_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + if (metaBuilder_ == null) { + meta_ = null; + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_File_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File + .getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File build() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File buildPartial() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File result = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.name_ = name_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + if (metaBuilder_ == null) { + result.meta_ = meta_; + } else { + result.meta_ = metaBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File other) { + if (other == com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File + .getDefaultInstance()) + return this; + if (other.hasName()) { + bitField0_ |= 0x00000001; + name_ = other.name_; + onChanged(); + } + if (other.hasMeta()) { + mergeMeta(other.getMeta()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasName()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object name_ = ""; + + /** + * required string name = 1; + */ + public boolean hasName() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string name = 1; + */ + public java.lang.String getName() { + java.lang.Object ref = name_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + name_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string name = 1; + */ + public com.google.protobuf.ByteString getNameBytes() { + java.lang.Object ref = name_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + name_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string name = 1; + */ + public Builder setName(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + + /** + * required string name = 1; + */ + public Builder clearName() { + bitField0_ = (bitField0_ & ~0x00000001); + name_ = getDefaultInstance().getName(); + onChanged(); + return this; + } + + /** + * required string name = 1; + */ + public Builder setNameBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + name_ = value; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta meta_ = null; + private com.google.protobuf.SingleFieldBuilderV3 metaBuilder_; + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta getMeta() { + if (metaBuilder_ == null) { + return meta_ == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta + .getDefaultInstance() : meta_; + } else { + return metaBuilder_.getMessage(); + } + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta value) { + if (metaBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + meta_ = value; + onChanged(); + } else { + metaBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder builderForValue) { + if (metaBuilder_ == null) { + meta_ = builderForValue.build(); + onChanged(); + } else { + metaBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public Builder mergeMeta(com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta value) { + if (metaBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) + && meta_ != null + && meta_ != com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta + .getDefaultInstance()) { + meta_ = com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.newBuilder(meta_) + .mergeFrom(value).buildPartial(); + } else { + meta_ = value; + } + onChanged(); + } else { + metaBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public Builder clearMeta() { + if (metaBuilder_ == null) { + meta_ = null; + onChanged(); + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder getMetaBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getMetaFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + public com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMetaOrBuilder getMetaOrBuilder() { + if (metaBuilder_ != null) { + return metaBuilder_.getMessageOrBuilder(); + } else { + return meta_ == null ? com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta + .getDefaultInstance() : meta_; + } + } + + /** + * optional .jraft.LocalFileMeta meta = 2; + */ + private com.google.protobuf.SingleFieldBuilderV3 getMetaFieldBuilder() { + if (metaBuilder_ == null) { + metaBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getMeta(), getParentForChildren(), isClean()); + meta_ = null; + } + return metaBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.LocalSnapshotPbMeta.File) + } + + // @@protoc_insertion_point(class_scope:jraft.LocalSnapshotPbMeta.File) + private static final com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File(); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public File parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new File(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private int bitField0_; + public static final int META_FIELD_NUMBER = 1; + private com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta meta_; + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta() { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() : meta_; + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder() { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() : meta_; + } + + public static final int FILES_FIELD_NUMBER = 2; + private java.util.List files_; + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public java.util.List getFilesList() { + return files_; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public java.util.List getFilesOrBuilderList() { + return files_; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public int getFilesCount() { + return files_.size(); + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getFiles(int index) { + return files_.get(index); + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.FileOrBuilder getFilesOrBuilder(int index) { + return files_.get(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasMeta()) { + if (!getMeta().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + for (int i = 0; i < getFilesCount(); i++) { + if (!getFiles(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(1, getMeta()); + } + for (int i = 0; i < files_.size(); i++) { + output.writeMessage(2, files_.get(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(1, getMeta()); + } + for (int i = 0; i < files_.size(); i++) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(2, files_.get(i)); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta other = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta) obj; + + boolean result = true; + result = result && (hasMeta() == other.hasMeta()); + if (hasMeta()) { + result = result && getMeta().equals(other.getMeta()); + } + result = result && getFilesList().equals(other.getFilesList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasMeta()) { + hash = (37 * hash) + META_FIELD_NUMBER; + hash = (53 * hash) + getMeta().hashCode(); + } + if (getFilesCount() > 0) { + hash = (37 * hash) + FILES_FIELD_NUMBER; + hash = (53 * hash) + getFilesList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.LocalSnapshotPbMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.LocalSnapshotPbMeta) + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.class, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getMetaFieldBuilder(); + getFilesFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + if (metaBuilder_ == null) { + meta_ = null; + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + if (filesBuilder_ == null) { + files_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + } else { + filesBuilder_.clear(); + } + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.internal_static_jraft_LocalSnapshotPbMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta build() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta buildPartial() { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta result = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + if (metaBuilder_ == null) { + result.meta_ = meta_; + } else { + result.meta_ = metaBuilder_.build(); + } + if (filesBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002)) { + files_ = java.util.Collections.unmodifiableList(files_); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.files_ = files_; + } else { + result.files_ = filesBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta other) { + if (other == com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.getDefaultInstance()) + return this; + if (other.hasMeta()) { + mergeMeta(other.getMeta()); + } + if (filesBuilder_ == null) { + if (!other.files_.isEmpty()) { + if (files_.isEmpty()) { + files_ = other.files_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureFilesIsMutable(); + files_.addAll(other.files_); + } + onChanged(); + } + } else { + if (!other.files_.isEmpty()) { + if (filesBuilder_.isEmpty()) { + filesBuilder_.dispose(); + filesBuilder_ = null; + files_ = other.files_; + bitField0_ = (bitField0_ & ~0x00000002); + filesBuilder_ = com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? getFilesFieldBuilder() + : null; + } else { + filesBuilder_.addAllMessages(other.files_); + } + } + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasMeta()) { + if (!getMeta().isInitialized()) { + return false; + } + } + for (int i = 0; i < getFilesCount(); i++) { + if (!getFiles(i).isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta meta_ = null; + private com.google.protobuf.SingleFieldBuilderV3 metaBuilder_; + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta() { + if (metaBuilder_ == null) { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() + : meta_; + } else { + return metaBuilder_.getMessage(); + } + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta value) { + if (metaBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + meta_ = value; + onChanged(); + } else { + metaBuilder_.setMessage(value); + } + bitField0_ |= 0x00000001; + return this; + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder builderForValue) { + if (metaBuilder_ == null) { + meta_ = builderForValue.build(); + onChanged(); + } else { + metaBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000001; + return this; + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public Builder mergeMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta value) { + if (metaBuilder_ == null) { + if (((bitField0_ & 0x00000001) == 0x00000001) && meta_ != null + && meta_ != com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance()) { + meta_ = com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.newBuilder(meta_).mergeFrom(value) + .buildPartial(); + } else { + meta_ = value; + } + onChanged(); + } else { + metaBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000001; + return this; + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public Builder clearMeta() { + if (metaBuilder_ == null) { + meta_ = null; + onChanged(); + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder getMetaBuilder() { + bitField0_ |= 0x00000001; + onChanged(); + return getMetaFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder() { + if (metaBuilder_ != null) { + return metaBuilder_.getMessageOrBuilder(); + } else { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() + : meta_; + } + } + + /** + * optional .jraft.SnapshotMeta meta = 1; + */ + private com.google.protobuf.SingleFieldBuilderV3 getMetaFieldBuilder() { + if (metaBuilder_ == null) { + metaBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getMeta(), getParentForChildren(), isClean()); + meta_ = null; + } + return metaBuilder_; + } + + private java.util.List files_ = java.util.Collections + .emptyList(); + + private void ensureFilesIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + files_ = new java.util.ArrayList( + files_); + bitField0_ |= 0x00000002; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3 filesBuilder_; + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public java.util.List getFilesList() { + if (filesBuilder_ == null) { + return java.util.Collections.unmodifiableList(files_); + } else { + return filesBuilder_.getMessageList(); + } + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public int getFilesCount() { + if (filesBuilder_ == null) { + return files_.size(); + } else { + return filesBuilder_.getCount(); + } + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File getFiles(int index) { + if (filesBuilder_ == null) { + return files_.get(index); + } else { + return filesBuilder_.getMessage(index); + } + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder setFiles(int index, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File value) { + if (filesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureFilesIsMutable(); + files_.set(index, value); + onChanged(); + } else { + filesBuilder_.setMessage(index, value); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder setFiles(int index, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder builderForValue) { + if (filesBuilder_ == null) { + ensureFilesIsMutable(); + files_.set(index, builderForValue.build()); + onChanged(); + } else { + filesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder addFiles(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File value) { + if (filesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureFilesIsMutable(); + files_.add(value); + onChanged(); + } else { + filesBuilder_.addMessage(value); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder addFiles(int index, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File value) { + if (filesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureFilesIsMutable(); + files_.add(index, value); + onChanged(); + } else { + filesBuilder_.addMessage(index, value); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder addFiles(com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder builderForValue) { + if (filesBuilder_ == null) { + ensureFilesIsMutable(); + files_.add(builderForValue.build()); + onChanged(); + } else { + filesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder addFiles(int index, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder builderForValue) { + if (filesBuilder_ == null) { + ensureFilesIsMutable(); + files_.add(index, builderForValue.build()); + onChanged(); + } else { + filesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder addAllFiles(java.lang.Iterable values) { + if (filesBuilder_ == null) { + ensureFilesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, files_); + onChanged(); + } else { + filesBuilder_.addAllMessages(values); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder clearFiles() { + if (filesBuilder_ == null) { + files_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + } else { + filesBuilder_.clear(); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public Builder removeFiles(int index) { + if (filesBuilder_ == null) { + ensureFilesIsMutable(); + files_.remove(index); + onChanged(); + } else { + filesBuilder_.remove(index); + } + return this; + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder getFilesBuilder(int index) { + return getFilesFieldBuilder().getBuilder(index); + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.FileOrBuilder getFilesOrBuilder(int index) { + if (filesBuilder_ == null) { + return files_.get(index); + } else { + return filesBuilder_.getMessageOrBuilder(index); + } + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public java.util.List getFilesOrBuilderList() { + if (filesBuilder_ != null) { + return filesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(files_); + } + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder addFilesBuilder() { + return getFilesFieldBuilder().addBuilder( + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.getDefaultInstance()); + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.Builder addFilesBuilder(int index) { + return getFilesFieldBuilder().addBuilder(index, + com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File.getDefaultInstance()); + } + + /** + * repeated .jraft.LocalSnapshotPbMeta.File files = 2; + */ + public java.util.List getFilesBuilderList() { + return getFilesFieldBuilder().getBuilderList(); + } + + private com.google.protobuf.RepeatedFieldBuilderV3 getFilesFieldBuilder() { + if (filesBuilder_ == null) { + filesBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3( + files_, ((bitField0_ & 0x00000002) == 0x00000002), getParentForChildren(), isClean()); + files_ = null; + } + return filesBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.LocalSnapshotPbMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.LocalSnapshotPbMeta) + private static final com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta(); + } + + public static com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public LocalSnapshotPbMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new LocalSnapshotPbMeta( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ConfigurationPBMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ConfigurationPBMeta_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_LogPBMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_LogPBMeta_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_StablePBMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_StablePBMeta_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_LocalSnapshotPbMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_LocalSnapshotPbMeta_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_LocalSnapshotPbMeta_File_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_LocalSnapshotPbMeta_File_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\023local_storage.proto\022\005jraft\032\nraft.proto" + + "\032\025local_file_meta.proto\"7\n\023Configuration" + + "PBMeta\022\r\n\005peers\030\001 \003(\t\022\021\n\told_peers\030\002 \003(\t" + + "\"$\n\tLogPBMeta\022\027\n\017first_log_index\030\001 \002(\003\"." + + "\n\014StablePBMeta\022\014\n\004term\030\001 \002(\003\022\020\n\010votedfor" + + "\030\002 \002(\t\"\242\001\n\023LocalSnapshotPbMeta\022!\n\004meta\030\001" + + " \001(\0132\023.jraft.SnapshotMeta\022.\n\005files\030\002 \003(\013" + + "2\037.jraft.LocalSnapshotPbMeta.File\0328\n\004Fil" + + "e\022\014\n\004name\030\001 \002(\t\022\"\n\004meta\030\002 \001(\0132\024.jraft.Lo" + + "calFileMetaB2\n\034com.alipay.sofa.jraft.ent" + + "ityB\022LocalStorageOutter" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom( + descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + com.alipay.sofa.jraft.entity.RaftOutter.getDescriptor(), + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.getDescriptor(), }, assigner); + internal_static_jraft_ConfigurationPBMeta_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_ConfigurationPBMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ConfigurationPBMeta_descriptor, new java.lang.String[] { "Peers", "OldPeers", }); + internal_static_jraft_LogPBMeta_descriptor = getDescriptor().getMessageTypes().get(1); + internal_static_jraft_LogPBMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_LogPBMeta_descriptor, new java.lang.String[] { "FirstLogIndex", }); + internal_static_jraft_StablePBMeta_descriptor = getDescriptor().getMessageTypes().get(2); + internal_static_jraft_StablePBMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_StablePBMeta_descriptor, new java.lang.String[] { "Term", "Votedfor", }); + internal_static_jraft_LocalSnapshotPbMeta_descriptor = getDescriptor().getMessageTypes().get(3); + internal_static_jraft_LocalSnapshotPbMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_LocalSnapshotPbMeta_descriptor, new java.lang.String[] { "Meta", "Files", }); + internal_static_jraft_LocalSnapshotPbMeta_File_descriptor = internal_static_jraft_LocalSnapshotPbMeta_descriptor + .getNestedTypes().get(0); + internal_static_jraft_LocalSnapshotPbMeta_File_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_LocalSnapshotPbMeta_File_descriptor, new java.lang.String[] { "Name", "Meta", }); + com.alipay.sofa.jraft.entity.RaftOutter.getDescriptor(); + com.alipay.sofa.jraft.entity.LocalFileMetaOutter.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogEntry.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogEntry.java new file mode 100644 index 0000000..5c82e4c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogEntry.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; +import com.alipay.sofa.jraft.entity.codec.v1.LogEntryV1CodecFactory; +import com.alipay.sofa.jraft.entity.codec.v1.V1Decoder; +import com.alipay.sofa.jraft.entity.codec.v1.V1Encoder; +import com.alipay.sofa.jraft.util.CrcUtil; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * A replica log entry. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:13:02 PM + */ +public class LogEntry implements Checksum { + + public static final ByteBuffer EMPTY_DATA = ByteBuffer.wrap(new byte[0]); + + /** entry type */ + private EnumOutter.EntryType type; + /** log id with index/term */ + private LogId id = new LogId(0, 0); + /** log entry current peers */ + private List peers; + /** log entry old peers */ + private List oldPeers; + /** log entry current learners */ + private List learners; + /** log entry old learners */ + private List oldLearners; + /** entry data */ + private ByteBuffer data = EMPTY_DATA; + /** checksum for log entry*/ + private long checksum; + /** true when the log has checksum **/ + private boolean hasChecksum; + + public List getLearners() { + return this.learners; + } + + public void setLearners(final List learners) { + this.learners = learners; + } + + public List getOldLearners() { + return this.oldLearners; + } + + public void setOldLearners(final List oldLearners) { + this.oldLearners = oldLearners; + } + + public LogEntry() { + super(); + } + + public LogEntry(final EnumOutter.EntryType type) { + super(); + this.type = type; + } + + public boolean hasLearners() { + return (this.learners != null && !this.learners.isEmpty()) + || (this.oldLearners != null && !this.oldLearners.isEmpty()); + } + + @Override + public long checksum() { + long c = checksum(this.type.getNumber(), this.id.checksum()); + c = checksum(this.peers, c); + c = checksum(this.oldPeers, c); + c = checksum(this.learners, c); + c = checksum(this.oldLearners, c); + if (this.data != null && this.data.hasRemaining()) { + c = checksum(c, CrcUtil.crc64(this.data)); + } + return c; + } + + /** + * Please use {@link LogEntryEncoder} instead. + * + * @deprecated + * @return encoded byte array + */ + @SuppressWarnings("DeprecatedIsStillUsed") + @Deprecated + public byte[] encode() { + return V1Encoder.INSTANCE.encode(this); + } + + /** + * Please use {@link LogEntryDecoder} instead. + * + * @deprecated + * @return whether success to decode + */ + @SuppressWarnings("DeprecatedIsStillUsed") + @Deprecated + public boolean decode(final byte[] content) { + if (content == null || content.length == 0) { + return false; + } + if (content[0] != LogEntryV1CodecFactory.MAGIC) { + // Corrupted log + return false; + } + V1Decoder.INSTANCE.decode(this, content); + return true; + } + + /** + * Returns whether the log entry has a checksum. + * @return true when the log entry has checksum, otherwise returns false. + * @since 1.2.26 + */ + public boolean hasChecksum() { + return this.hasChecksum; + } + + /** + * Returns true when the log entry is corrupted, it means that the checksum is mismatch. + * @since 1.2.6 + * @return true when the log entry is corrupted, otherwise returns false + */ + public boolean isCorrupted() { + return this.hasChecksum && this.checksum != checksum(); + } + + /** + * Returns the checksum of the log entry. You should use {@link #hasChecksum} to check if + * it has checksum. + * @return checksum value + */ + public long getChecksum() { + return this.checksum; + } + + public void setChecksum(final long checksum) { + this.checksum = checksum; + this.hasChecksum = true; + } + + public EnumOutter.EntryType getType() { + return this.type; + } + + public void setType(final EnumOutter.EntryType type) { + this.type = type; + } + + public LogId getId() { + return this.id; + } + + public void setId(final LogId id) { + this.id = id; + } + + public List getPeers() { + return this.peers; + } + + public void setPeers(final List peers) { + this.peers = peers; + } + + public List getOldPeers() { + return this.oldPeers; + } + + public void setOldPeers(final List oldPeers) { + this.oldPeers = oldPeers; + } + + /** + * Returns the log data, it's not read-only, you SHOULD take care it's modification and + * thread-safety by yourself. + * + * @return the log data + */ + public ByteBuffer getData() { + return this.data; + } + + /** + * Creates a new byte buffer whose content is a shared subsequence of this log entry's data + * buffer's content. + * + * @return The new byte buffer + */ + public ByteBuffer sliceData() { + return this.data != null ? this.data.slice() : null; + } + + /** + * Creates a new, read-only byte buffer that shares this log entry's data buffer's content. + * + * @return the new, read-only byte buffer + */ + public ByteBuffer getReadOnlyData() { + return this.data != null ? this.data.asReadOnlyBuffer() : null; + } + + public void setData(final ByteBuffer data) { + this.data = data; + } + + @Override + public String toString() { + return "LogEntry [type=" + this.type + ", id=" + this.id + ", peers=" + this.peers + ", oldPeers=" + + this.oldPeers + ", learners=" + this.learners + ", oldLearners=" + this.oldLearners + ", data=" + + (this.data != null ? this.data.remaining() : 0) + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((this.data == null) ? 0 : this.data.hashCode()); + result = prime * result + ((this.id == null) ? 0 : this.id.hashCode()); + result = prime * result + ((this.learners == null) ? 0 : this.learners.hashCode()); + result = prime * result + ((this.oldLearners == null) ? 0 : this.oldLearners.hashCode()); + result = prime * result + ((this.oldPeers == null) ? 0 : this.oldPeers.hashCode()); + result = prime * result + ((this.peers == null) ? 0 : this.peers.hashCode()); + result = prime * result + ((this.type == null) ? 0 : this.type.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + LogEntry other = (LogEntry) obj; + if (this.data == null) { + if (other.data != null) { + return false; + } + } else if (!this.data.equals(other.data)) { + return false; + } + if (this.id == null) { + if (other.id != null) { + return false; + } + } else if (!this.id.equals(other.id)) { + return false; + } + if (this.learners == null) { + if (other.learners != null) { + return false; + } + } else if (!this.learners.equals(other.learners)) { + return false; + } + if (this.oldLearners == null) { + if (other.oldLearners != null) { + return false; + } + } else if (!this.oldLearners.equals(other.oldLearners)) { + return false; + } + if (this.oldPeers == null) { + if (other.oldPeers != null) { + return false; + } + } else if (!this.oldPeers.equals(other.oldPeers)) { + return false; + } + if (this.peers == null) { + if (other.peers != null) { + return false; + } + } else if (!this.peers.equals(other.peers)) { + return false; + } + return this.type == other.type; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogId.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogId.java new file mode 100644 index 0000000..496db03 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/LogId.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.io.Serializable; + +import com.alipay.sofa.jraft.util.Bits; +import com.alipay.sofa.jraft.util.Copiable; +import com.alipay.sofa.jraft.util.CrcUtil; + +/** + * Log identifier. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:12:29 PM + */ +public class LogId implements Comparable, Copiable, Serializable, Checksum { + + private static final long serialVersionUID = -6680425579347357313L; + + private long index; + private long term; + + @Override + public LogId copy() { + return new LogId(this.index, this.term); + } + + @Override + public long checksum() { + byte[] bs = new byte[16]; + Bits.putLong(bs, 0, this.index); + Bits.putLong(bs, 8, this.term); + return CrcUtil.crc64(bs); + } + + public LogId() { + this(0, 0); + } + + public LogId(final long index, final long term) { + super(); + setIndex(index); + setTerm(term); + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (int) (this.index ^ (this.index >>> 32)); + result = prime * result + (int) (this.term ^ (this.term >>> 32)); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final LogId other = (LogId) obj; + if (this.index != other.index) { + return false; + } + // noinspection RedundantIfStatement + if (this.term != other.term) { + return false; + } + return true; + } + + @Override + public int compareTo(final LogId o) { + // Compare term at first + final int c = Long.compare(getTerm(), o.getTerm()); + if (c == 0) { + return Long.compare(getIndex(), o.getIndex()); + } else { + return c; + } + } + + public long getTerm() { + return this.term; + } + + public void setTerm(final long term) { + this.term = term; + } + + public long getIndex() { + return this.index; + } + + public void setIndex(final long index) { + this.index = index; + } + + @Override + public String toString() { + return "LogId [index=" + this.index + ", term=" + this.term + "]"; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/NodeId.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/NodeId.java new file mode 100644 index 0000000..3d835a3 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/NodeId.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.io.Serializable; + +/** + * A raft node identifier. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:08:14 PM + */ +public final class NodeId implements Serializable { + + private static final long serialVersionUID = 4428173460056804264L; + + /** Raft group id*/ + private final String groupId; + /** Node peer id*/ + private final PeerId peerId; + /** cached toString result*/ + private String str; + + public NodeId(String groupId, PeerId peerId) { + super(); + this.groupId = groupId; + this.peerId = peerId; + } + + public String getGroupId() { + return this.groupId; + } + + @Override + public String toString() { + if (str == null) { + str = "<" + this.groupId + "/" + this.peerId + ">"; + } + return str; + } + + public PeerId getPeerId() { + return this.peerId; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (this.groupId == null ? 0 : this.groupId.hashCode()); + result = prime * result + (this.peerId == null ? 0 : this.peerId.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final NodeId other = (NodeId) obj; + if (this.groupId == null) { + if (other.groupId != null) { + return false; + } + } else if (!this.groupId.equals(other.groupId)) { + return false; + } + if (this.peerId == null) { + return other.peerId == null; + } else { + return this.peerId.equals(other.peerId); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/PeerId.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/PeerId.java new file mode 100644 index 0000000..6851f16 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/PeerId.java @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.io.Serializable; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.core.ElectionPriority; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.alipay.sofa.jraft.util.Copiable; +import com.alipay.sofa.jraft.util.CrcUtil; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Represent a participant in a replicating group. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:27:37 PM + */ +public class PeerId implements Copiable, Serializable, Checksum { + + private static final long serialVersionUID = 8083529734784884641L; + + private static final Logger LOG = LoggerFactory.getLogger(PeerId.class); + + /** Peer address. */ + private Endpoint endpoint = new Endpoint(Utils.IP_ANY, 0); + /** Index in same addr, default is 0. */ + private int idx; + /** Cached toString result. */ + private String str; + + /** Node's local priority value, if node don't support priority election, this value is -1. */ + private int priority = ElectionPriority.Disabled; + + public static final PeerId ANY_PEER = new PeerId(); + + private long checksum; + + public PeerId() { + super(); + } + + @Override + public long checksum() { + if (this.checksum == 0) { + this.checksum = CrcUtil.crc64(AsciiStringUtil.unsafeEncode(toString())); + } + return this.checksum; + } + + /** + * Create an empty peer. + * @return empty peer + */ + public static PeerId emptyPeer() { + return new PeerId(); + } + + @Override + public PeerId copy() { + return new PeerId(this.endpoint.copy(), this.idx, this.priority); + } + + /** + * Parse a peer from string in the format of "ip:port:idx", + * returns null if fail to parse. + * + * @param s input string with the format of "ip:port:idx" + * @return parsed peer + */ + public static PeerId parsePeer(final String s) { + final PeerId peer = new PeerId(); + if (peer.parse(s)) { + return peer; + } + return null; + } + + public PeerId(final Endpoint endpoint, final int idx) { + super(); + this.endpoint = endpoint; + this.idx = idx; + } + + public PeerId(final String ip, final int port) { + this(ip, port, 0); + } + + public PeerId(final String ip, final int port, final int idx) { + super(); + this.endpoint = new Endpoint(ip, port); + this.idx = idx; + } + + public PeerId(final Endpoint endpoint, final int idx, final int priority) { + super(); + this.endpoint = endpoint; + this.idx = idx; + this.priority = priority; + } + + public PeerId(final String ip, final int port, final int idx, final int priority) { + super(); + this.endpoint = new Endpoint(ip, port); + this.idx = idx; + this.priority = priority; + } + + public Endpoint getEndpoint() { + return this.endpoint; + } + + public String getIp() { + return this.endpoint.getIp(); + } + + public int getPort() { + return this.endpoint.getPort(); + } + + public int getIdx() { + return this.idx; + } + + public int getPriority() { + return priority; + } + + public void setPriority(int priority) { + this.priority = priority; + this.str = null; + } + + /** + * Returns true when ip is ANY_IP, port is zero and idx is zero too. + */ + public boolean isEmpty() { + return getIp().equals(Utils.IP_ANY) && getPort() == 0 && this.idx == 0; + } + + @Override + public String toString() { + if (this.str == null) { + final StringBuilder buf = new StringBuilder(this.endpoint.toString()); + + if (this.idx != 0) { + buf.append(':').append(this.idx); + } + + if (this.priority != ElectionPriority.Disabled) { + if (this.idx == 0) { + buf.append(':'); + } + buf.append(':').append(this.priority); + } + + this.str = buf.toString(); + } + return this.str; + } + + /** + * Parse peerId from string that generated by {@link #toString()} + * This method can support parameter string values are below: + * + * 
+     * PeerId.parse("a:b")          = new PeerId("a", "b", 0 , -1)
+     * PeerId.parse("a:b:c")        = new PeerId("a", "b", "c", -1)
+     * PeerId.parse("a:b::d")       = new PeerId("a", "b", 0, "d")
+     * PeerId.parse("a:b:c:d")      = new PeerId("a", "b", "c", "d")
+     *
+ * + */ + public boolean parse(final String s) { + if (StringUtils.isEmpty(s)) { + return false; + } + + final String[] tmps = Utils.parsePeerId(s); + if (tmps.length < 2 || tmps.length > 4) { + return false; + } + try { + final int port = Integer.parseInt(tmps[1]); + this.endpoint = new Endpoint(tmps[0], port); + + switch (tmps.length) { + case 3: + this.idx = Integer.parseInt(tmps[2]); + break; + case 4: + if (tmps[2].equals("")) { + this.idx = 0; + } else { + this.idx = Integer.parseInt(tmps[2]); + } + this.priority = Integer.parseInt(tmps[3]); + break; + default: + break; + } + this.str = null; + return true; + } catch (final Exception e) { + LOG.error("Parse peer from string failed: {}.", s, e); + return false; + } + } + + /** + * To judge whether this node can participate in election or not. + * + * @return the restul that whether this node can participate in election or not. + */ + public boolean isPriorityNotElected() { + return this.priority == ElectionPriority.NotElected; + } + + /** + * To judge whether the priority election function is disabled or not in this node. + * + * @return the result that whether this node has priority election function or not. + */ + public boolean isPriorityDisabled() { + return this.priority <= ElectionPriority.Disabled; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + (this.endpoint == null ? 0 : this.endpoint.hashCode()); + result = prime * result + this.idx; + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final PeerId other = (PeerId) obj; + if (this.endpoint == null) { + if (other.endpoint != null) { + return false; + } + } else if (!this.endpoint.equals(other.endpoint)) { + return false; + } + return this.idx == other.idx; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/RaftOutter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/RaftOutter.java new file mode 100644 index 0000000..4f39aa0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/RaftOutter.java @@ -0,0 +1,3114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: raft.proto + +package com.alipay.sofa.jraft.entity; + +public final class RaftOutter { + private RaftOutter() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public interface EntryMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.EntryMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required .jraft.EntryType type = 2; + */ + boolean hasType(); + + /** + * required .jraft.EntryType type = 2; + */ + com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType(); + + /** + * repeated string peers = 3; + */ + java.util.List getPeersList(); + + /** + * repeated string peers = 3; + */ + int getPeersCount(); + + /** + * repeated string peers = 3; + */ + java.lang.String getPeers(int index); + + /** + * repeated string peers = 3; + */ + com.google.protobuf.ByteString getPeersBytes(int index); + + /** + * optional int64 data_len = 4; + */ + boolean hasDataLen(); + + /** + * optional int64 data_len = 4; + */ + long getDataLen(); + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + java.util.List getOldPeersList(); + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + int getOldPeersCount(); + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + java.lang.String getOldPeers(int index); + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * 
+         * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+         *
+ * + * optional int64 checksum = 6; + */ + boolean hasChecksum(); + + /** + * 
+         * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+         *
+ * + * optional int64 checksum = 6; + */ + long getChecksum(); + + /** + * repeated string learners = 7; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 7; + */ + int getLearnersCount(); + + /** + * repeated string learners = 7; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 7; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + + /** + * repeated string old_learners = 8; + */ + java.util.List getOldLearnersList(); + + /** + * repeated string old_learners = 8; + */ + int getOldLearnersCount(); + + /** + * repeated string old_learners = 8; + */ + java.lang.String getOldLearners(int index); + + /** + * repeated string old_learners = 8; + */ + com.google.protobuf.ByteString getOldLearnersBytes(int index); + } + + /** + * Protobuf type {@code jraft.EntryMeta} + */ + public static final class EntryMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.EntryMeta) + EntryMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use EntryMeta.newBuilder() to construct. + private EntryMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private EntryMeta() { + term_ = 0L; + type_ = 0; + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + dataLen_ = 0L; + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + checksum_ = 0L; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private EntryMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 16: { + int rawValue = input.readEnum(); + com.alipay.sofa.jraft.entity.EnumOutter.EntryType value = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(2, rawValue); + } else { + bitField0_ |= 0x00000002; + type_ = rawValue; + } + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + peers_.add(bs); + break; + } + case 32: { + bitField0_ |= 0x00000004; + dataLen_ = input.readInt64(); + break; + } + case 42: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000010; + } + oldPeers_.add(bs); + break; + } + case 48: { + bitField0_ |= 0x00000008; + checksum_ = input.readInt64(); + break; + } + case 58: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000040; + } + learners_.add(bs); + break; + } + case 66: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000080) == 0x00000080)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000080; + } + oldLearners_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = peers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + learners_ = learners_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000080) == 0x00000080)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_EntryMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_EntryMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.class, + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int TYPE_FIELD_NUMBER = 2; + private int type_; + + /** + * required .jraft.EntryType type = 2; + */ + public boolean hasType() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required .jraft.EntryType type = 2; + */ + public com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType() { + com.alipay.sofa.jraft.entity.EnumOutter.EntryType result = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(type_); + return result == null ? com.alipay.sofa.jraft.entity.EnumOutter.EntryType.ENTRY_TYPE_UNKNOWN : result; + } + + public static final int PEERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList peers_; + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_; + } + + /** + * repeated string peers = 3; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 3; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + public static final int DATA_LEN_FIELD_NUMBER = 4; + private long dataLen_; + + /** + * optional int64 data_len = 4; + */ + public boolean hasDataLen() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional int64 data_len = 4; + */ + public long getDataLen() { + return dataLen_; + } + + public static final int OLD_PEERS_FIELD_NUMBER = 5; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * 
+         * Don't change field id of `old_peers' in the consideration of backward
+         * compatibility
+         *
+ * + * repeated string old_peers = 5; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int CHECKSUM_FIELD_NUMBER = 6; + private long checksum_; + + /** + * 
+         * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+         *
+ * + * optional int64 checksum = 6; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * 
+         * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+         *
+ * + * optional int64 checksum = 6; + */ + public long getChecksum() { + return checksum_; + } + + public static final int LEARNERS_FIELD_NUMBER = 7; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 7; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 7; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 7; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 7; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + public static final int OLD_LEARNERS_FIELD_NUMBER = 8; + private com.google.protobuf.LazyStringList oldLearners_; + + /** + * repeated string old_learners = 8; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_; + } + + /** + * repeated string old_learners = 8; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 8; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 8; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasType()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeEnum(2, type_); + } + for (int i = 0; i < peers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peers_.getRaw(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(4, dataLen_); + } + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 5, oldPeers_.getRaw(i)); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(6, checksum_); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 7, learners_.getRaw(i)); + } + for (int i = 0; i < oldLearners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 8, oldLearners_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeEnumSize(2, type_); + } + { + int dataSize = 0; + for (int i = 0; i < peers_.size(); i++) { + dataSize += computeStringSizeNoTag(peers_.getRaw(i)); + } + size += dataSize; + size += 1 * getPeersList().size(); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, dataLen_); + } + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(6, checksum_); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldLearners_.size(); i++) { + dataSize += computeStringSizeNoTag(oldLearners_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta other = (com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasType() == other.hasType()); + if (hasType()) { + result = result && type_ == other.type_; + } + result = result && getPeersList().equals(other.getPeersList()); + result = result && (hasDataLen() == other.hasDataLen()); + if (hasDataLen()) { + result = result && (getDataLen() == other.getDataLen()); + } + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && (hasChecksum() == other.hasChecksum()); + if (hasChecksum()) { + result = result && (getChecksum() == other.getChecksum()); + } + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && getOldLearnersList().equals(other.getOldLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasType()) { + hash = (37 * hash) + TYPE_FIELD_NUMBER; + hash = (53 * hash) + type_; + } + if (getPeersCount() > 0) { + hash = (37 * hash) + PEERS_FIELD_NUMBER; + hash = (53 * hash) + getPeersList().hashCode(); + } + if (hasDataLen()) { + hash = (37 * hash) + DATA_LEN_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getDataLen()); + } + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (hasChecksum()) { + hash = (37 * hash) + CHECKSUM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getChecksum()); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + if (getOldLearnersCount() > 0) { + hash = (37 * hash) + OLD_LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getOldLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.EntryMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.EntryMeta) + com.alipay.sofa.jraft.entity.RaftOutter.EntryMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_EntryMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_EntryMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.class, + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + type_ = 0; + bitField0_ = (bitField0_ & ~0x00000002); + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + dataLen_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000010); + checksum_ = 0L; + bitField0_ = (bitField0_ & ~0x00000020); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000040); + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000080); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_EntryMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta build() { + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta buildPartial() { + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta result = new com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.type_ = type_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = peers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.peers_ = peers_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.dataLen_ = dataLen_; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.oldPeers_ = oldPeers_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000008; + } + result.checksum_ = checksum_; + if (((bitField0_ & 0x00000040) == 0x00000040)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000040); + } + result.learners_ = learners_; + if (((bitField0_ & 0x00000080) == 0x00000080)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000080); + } + result.oldLearners_ = oldLearners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta other) { + if (other == com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasType()) { + setType(other.getType()); + } + if (!other.peers_.isEmpty()) { + if (peers_.isEmpty()) { + peers_ = other.peers_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensurePeersIsMutable(); + peers_.addAll(other.peers_); + } + onChanged(); + } + if (other.hasDataLen()) { + setDataLen(other.getDataLen()); + } + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (other.hasChecksum()) { + setChecksum(other.getChecksum()); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000040); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + if (!other.oldLearners_.isEmpty()) { + if (oldLearners_.isEmpty()) { + oldLearners_ = other.oldLearners_; + bitField0_ = (bitField0_ & ~0x00000080); + } else { + ensureOldLearnersIsMutable(); + oldLearners_.addAll(other.oldLearners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasType()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private int type_ = 0; + + /** + * required .jraft.EntryType type = 2; + */ + public boolean hasType() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required .jraft.EntryType type = 2; + */ + public com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType() { + com.alipay.sofa.jraft.entity.EnumOutter.EntryType result = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(type_); + return result == null ? com.alipay.sofa.jraft.entity.EnumOutter.EntryType.ENTRY_TYPE_UNKNOWN : result; + } + + /** + * required .jraft.EntryType type = 2; + */ + public Builder setType(com.alipay.sofa.jraft.entity.EnumOutter.EntryType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + type_ = value.getNumber(); + onChanged(); + return this; + } + + /** + * required .jraft.EntryType type = 2; + */ + public Builder clearType() { + bitField0_ = (bitField0_ & ~0x00000002); + type_ = 0; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensurePeersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = new com.google.protobuf.LazyStringArrayList(peers_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_.getUnmodifiableView(); + } + + /** + * repeated string peers = 3; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 3; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + /** + * repeated string peers = 3; + */ + public Builder setPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addAllPeers(java.lang.Iterable values) { + ensurePeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, peers_); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder clearPeers() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + private long dataLen_; + + /** + * optional int64 data_len = 4; + */ + public boolean hasDataLen() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional int64 data_len = 4; + */ + public long getDataLen() { + return dataLen_; + } + + /** + * optional int64 data_len = 4; + */ + public Builder setDataLen(long value) { + bitField0_ |= 0x00000008; + dataLen_ = value; + onChanged(); + return this; + } + + /** + * optional int64 data_len = 4; + */ + public Builder clearDataLen() { + bitField0_ = (bitField0_ & ~0x00000008); + dataLen_ = 0L; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000010; + } + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000010); + onChanged(); + return this; + } + + /** + * 
+             * Don't change field id of `old_peers' in the consideration of backward
+             * compatibility
+             *
+ * + * repeated string old_peers = 5; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private long checksum_; + + /** + * 
+             * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+             *
+ * + * optional int64 checksum = 6; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * 
+             * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+             *
+ * + * optional int64 checksum = 6; + */ + public long getChecksum() { + return checksum_; + } + + /** + * 
+             * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+             *
+ * + * optional int64 checksum = 6; + */ + public Builder setChecksum(long value) { + bitField0_ |= 0x00000020; + checksum_ = value; + onChanged(); + return this; + } + + /** + * 
+             * Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+             *
+ * + * optional int64 checksum = 6; + */ + public Builder clearChecksum() { + bitField0_ = (bitField0_ & ~0x00000020); + checksum_ = 0L; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000040) == 0x00000040)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000040; + } + } + + /** + * repeated string learners = 7; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 7; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 7; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 7; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 7; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 7; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 7; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 7; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000040); + onChanged(); + return this; + } + + /** + * repeated string learners = 7; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldLearnersIsMutable() { + if (!((bitField0_ & 0x00000080) == 0x00000080)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(oldLearners_); + bitField0_ |= 0x00000080; + } + } + + /** + * repeated string old_learners = 8; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_.getUnmodifiableView(); + } + + /** + * repeated string old_learners = 8; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 8; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 8; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + /** + * repeated string old_learners = 8; + */ + public Builder setOldLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 8; + */ + public Builder addOldLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 8; + */ + public Builder addAllOldLearners(java.lang.Iterable values) { + ensureOldLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldLearners_); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 8; + */ + public Builder clearOldLearners() { + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000080); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 8; + */ + public Builder addOldLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.EntryMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.EntryMeta) + private static final com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta(); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public EntryMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new EntryMeta(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface SnapshotMetaOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.SnapshotMeta) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 last_included_index = 1; + */ + boolean hasLastIncludedIndex(); + + /** + * required int64 last_included_index = 1; + */ + long getLastIncludedIndex(); + + /** + * required int64 last_included_term = 2; + */ + boolean hasLastIncludedTerm(); + + /** + * required int64 last_included_term = 2; + */ + long getLastIncludedTerm(); + + /** + * repeated string peers = 3; + */ + java.util.List getPeersList(); + + /** + * repeated string peers = 3; + */ + int getPeersCount(); + + /** + * repeated string peers = 3; + */ + java.lang.String getPeers(int index); + + /** + * repeated string peers = 3; + */ + com.google.protobuf.ByteString getPeersBytes(int index); + + /** + * repeated string old_peers = 4; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 4; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 4; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 4; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * repeated string learners = 5; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 5; + */ + int getLearnersCount(); + + /** + * repeated string learners = 5; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 5; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + + /** + * repeated string old_learners = 6; + */ + java.util.List getOldLearnersList(); + + /** + * repeated string old_learners = 6; + */ + int getOldLearnersCount(); + + /** + * repeated string old_learners = 6; + */ + java.lang.String getOldLearners(int index); + + /** + * repeated string old_learners = 6; + */ + com.google.protobuf.ByteString getOldLearnersBytes(int index); + } + + /** + * Protobuf type {@code jraft.SnapshotMeta} + */ + public static final class SnapshotMeta extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.SnapshotMeta) + SnapshotMetaOrBuilder { + private static final long serialVersionUID = 0L; + + // Use SnapshotMeta.newBuilder() to construct. + private SnapshotMeta(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private SnapshotMeta() { + lastIncludedIndex_ = 0L; + lastIncludedTerm_ = 0L; + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private SnapshotMeta(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + lastIncludedIndex_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + lastIncludedTerm_ = input.readInt64(); + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + peers_.add(bs); + break; + } + case 34: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000008; + } + oldPeers_.add(bs); + break; + } + case 42: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000010; + } + learners_.add(bs); + break; + } + case 50: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000020) == 0x00000020)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000020; + } + oldLearners_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = peers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + learners_ = learners_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000020) == 0x00000020)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_SnapshotMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_SnapshotMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.class, + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder.class); + } + + private int bitField0_; + public static final int LAST_INCLUDED_INDEX_FIELD_NUMBER = 1; + private long lastIncludedIndex_; + + /** + * required int64 last_included_index = 1; + */ + public boolean hasLastIncludedIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 last_included_index = 1; + */ + public long getLastIncludedIndex() { + return lastIncludedIndex_; + } + + public static final int LAST_INCLUDED_TERM_FIELD_NUMBER = 2; + private long lastIncludedTerm_; + + /** + * required int64 last_included_term = 2; + */ + public boolean hasLastIncludedTerm() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required int64 last_included_term = 2; + */ + public long getLastIncludedTerm() { + return lastIncludedTerm_; + } + + public static final int PEERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList peers_; + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_; + } + + /** + * repeated string peers = 3; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 3; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + public static final int OLD_PEERS_FIELD_NUMBER = 4; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 4; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 4; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 4; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 4; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int LEARNERS_FIELD_NUMBER = 5; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 5; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 5; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 5; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 5; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + public static final int OLD_LEARNERS_FIELD_NUMBER = 6; + private com.google.protobuf.LazyStringList oldLearners_; + + /** + * repeated string old_learners = 6; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_; + } + + /** + * repeated string old_learners = 6; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 6; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 6; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasLastIncludedIndex()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLastIncludedTerm()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, lastIncludedIndex_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, lastIncludedTerm_); + } + for (int i = 0; i < peers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peers_.getRaw(i)); + } + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 4, oldPeers_.getRaw(i)); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 5, learners_.getRaw(i)); + } + for (int i = 0; i < oldLearners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 6, oldLearners_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, lastIncludedIndex_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(2, lastIncludedTerm_); + } + { + int dataSize = 0; + for (int i = 0; i < peers_.size(); i++) { + dataSize += computeStringSizeNoTag(peers_.getRaw(i)); + } + size += dataSize; + size += 1 * getPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldLearners_.size(); i++) { + dataSize += computeStringSizeNoTag(oldLearners_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta other = (com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta) obj; + + boolean result = true; + result = result && (hasLastIncludedIndex() == other.hasLastIncludedIndex()); + if (hasLastIncludedIndex()) { + result = result && (getLastIncludedIndex() == other.getLastIncludedIndex()); + } + result = result && (hasLastIncludedTerm() == other.hasLastIncludedTerm()); + if (hasLastIncludedTerm()) { + result = result && (getLastIncludedTerm() == other.getLastIncludedTerm()); + } + result = result && getPeersList().equals(other.getPeersList()); + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && getOldLearnersList().equals(other.getOldLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasLastIncludedIndex()) { + hash = (37 * hash) + LAST_INCLUDED_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getLastIncludedIndex()); + } + if (hasLastIncludedTerm()) { + hash = (37 * hash) + LAST_INCLUDED_TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getLastIncludedTerm()); + } + if (getPeersCount() > 0) { + hash = (37 * hash) + PEERS_FIELD_NUMBER; + hash = (53 * hash) + getPeersList().hashCode(); + } + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + if (getOldLearnersCount() > 0) { + hash = (37 * hash) + OLD_LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getOldLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.SnapshotMeta} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.SnapshotMeta) + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_SnapshotMeta_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_SnapshotMeta_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.class, + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + lastIncludedIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + lastIncludedTerm_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000008); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000010); + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.RaftOutter.internal_static_jraft_SnapshotMeta_descriptor; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta build() { + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta buildPartial() { + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta result = new com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.lastIncludedIndex_ = lastIncludedIndex_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.lastIncludedTerm_ = lastIncludedTerm_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = peers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.peers_ = peers_; + if (((bitField0_ & 0x00000008) == 0x00000008)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000008); + } + result.oldPeers_ = oldPeers_; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.learners_ = learners_; + if (((bitField0_ & 0x00000020) == 0x00000020)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000020); + } + result.oldLearners_ = oldLearners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta) { + return mergeFrom((com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta other) { + if (other == com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance()) + return this; + if (other.hasLastIncludedIndex()) { + setLastIncludedIndex(other.getLastIncludedIndex()); + } + if (other.hasLastIncludedTerm()) { + setLastIncludedTerm(other.getLastIncludedTerm()); + } + if (!other.peers_.isEmpty()) { + if (peers_.isEmpty()) { + peers_ = other.peers_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensurePeersIsMutable(); + peers_.addAll(other.peers_); + } + onChanged(); + } + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000008); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + if (!other.oldLearners_.isEmpty()) { + if (oldLearners_.isEmpty()) { + oldLearners_ = other.oldLearners_; + bitField0_ = (bitField0_ & ~0x00000020); + } else { + ensureOldLearnersIsMutable(); + oldLearners_.addAll(other.oldLearners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasLastIncludedIndex()) { + return false; + } + if (!hasLastIncludedTerm()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long lastIncludedIndex_; + + /** + * required int64 last_included_index = 1; + */ + public boolean hasLastIncludedIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 last_included_index = 1; + */ + public long getLastIncludedIndex() { + return lastIncludedIndex_; + } + + /** + * required int64 last_included_index = 1; + */ + public Builder setLastIncludedIndex(long value) { + bitField0_ |= 0x00000001; + lastIncludedIndex_ = value; + onChanged(); + return this; + } + + /** + * required int64 last_included_index = 1; + */ + public Builder clearLastIncludedIndex() { + bitField0_ = (bitField0_ & ~0x00000001); + lastIncludedIndex_ = 0L; + onChanged(); + return this; + } + + private long lastIncludedTerm_; + + /** + * required int64 last_included_term = 2; + */ + public boolean hasLastIncludedTerm() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required int64 last_included_term = 2; + */ + public long getLastIncludedTerm() { + return lastIncludedTerm_; + } + + /** + * required int64 last_included_term = 2; + */ + public Builder setLastIncludedTerm(long value) { + bitField0_ |= 0x00000002; + lastIncludedTerm_ = value; + onChanged(); + return this; + } + + /** + * required int64 last_included_term = 2; + */ + public Builder clearLastIncludedTerm() { + bitField0_ = (bitField0_ & ~0x00000002); + lastIncludedTerm_ = 0L; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensurePeersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + peers_ = new com.google.protobuf.LazyStringArrayList(peers_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_.getUnmodifiableView(); + } + + /** + * repeated string peers = 3; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 3; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 3; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + /** + * repeated string peers = 3; + */ + public Builder setPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addAllPeers(java.lang.Iterable values) { + ensurePeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, peers_); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder clearPeers() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string peers = 3; + */ + public Builder addPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000008) == 0x00000008)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000008; + } + } + + /** + * repeated string old_peers = 4; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 4; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 4; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 4; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 4; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 4; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 4; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 4; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000008); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 4; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000010; + } + } + + /** + * repeated string learners = 5; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 5; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 5; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 5; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 5; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 5; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 5; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 5; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000010); + onChanged(); + return this; + } + + /** + * repeated string learners = 5; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldLearnersIsMutable() { + if (!((bitField0_ & 0x00000020) == 0x00000020)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(oldLearners_); + bitField0_ |= 0x00000020; + } + } + + /** + * repeated string old_learners = 6; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_.getUnmodifiableView(); + } + + /** + * repeated string old_learners = 6; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 6; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 6; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + /** + * repeated string old_learners = 6; + */ + public Builder setOldLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 6; + */ + public Builder addOldLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 6; + */ + public Builder addAllOldLearners(java.lang.Iterable values) { + ensureOldLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldLearners_); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 6; + */ + public Builder clearOldLearners() { + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 6; + */ + public Builder addOldLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.SnapshotMeta) + } + + // @@protoc_insertion_point(class_scope:jraft.SnapshotMeta) + private static final com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta(); + } + + public static com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public SnapshotMeta parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new SnapshotMeta(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_EntryMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_EntryMeta_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_SnapshotMeta_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_SnapshotMeta_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\nraft.proto\022\005jraft\032\nenum.proto\"\247\001\n\tEntr" + + "yMeta\022\014\n\004term\030\001 \002(\003\022\036\n\004type\030\002 \002(\0162\020.jraf" + + "t.EntryType\022\r\n\005peers\030\003 \003(\t\022\020\n\010data_len\030\004" + + " \001(\003\022\021\n\told_peers\030\005 \003(\t\022\020\n\010checksum\030\006 \001(" + + "\003\022\020\n\010learners\030\007 \003(\t\022\024\n\014old_learners\030\010 \003(" + + "\t\"\221\001\n\014SnapshotMeta\022\033\n\023last_included_inde" + + "x\030\001 \002(\003\022\032\n\022last_included_term\030\002 \002(\003\022\r\n\005p" + + "eers\030\003 \003(\t\022\021\n\told_peers\030\004 \003(\t\022\020\n\010learner" + + "s\030\005 \003(\t\022\024\n\014old_learners\030\006 \003(\tB*\n\034com.ali" + + "pay.sofa.jraft.entityB\nRaftOutter" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { com.alipay.sofa.jraft.entity.EnumOutter + .getDescriptor(), }, assigner); + internal_static_jraft_EntryMeta_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_EntryMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_EntryMeta_descriptor, new java.lang.String[] { "Term", "Type", "Peers", "DataLen", + "OldPeers", "Checksum", "Learners", "OldLearners", }); + internal_static_jraft_SnapshotMeta_descriptor = getDescriptor().getMessageTypes().get(1); + internal_static_jraft_SnapshotMeta_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_SnapshotMeta_descriptor, new java.lang.String[] { "LastIncludedIndex", + "LastIncludedTerm", "Peers", "OldPeers", "Learners", "OldLearners", }); + com.alipay.sofa.jraft.entity.EnumOutter.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexState.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexState.java new file mode 100644 index 0000000..80a1ea9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexState.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import com.alipay.sofa.jraft.closure.ReadIndexClosure; +import com.alipay.sofa.jraft.util.Bytes; + +/** + * ReadIndex state + * @author dennis + * + */ +public class ReadIndexState { + + /** The committed log index*/ + private long index = -1; + /** User request context*/ + private final Bytes requestContext; + /** User ReadIndex closure*/ + private final ReadIndexClosure done; + /** Request start timestamp*/ + private final long startTimeMs; + + public ReadIndexState(Bytes requestContext, ReadIndexClosure done, long startTimeMs) { + super(); + this.requestContext = requestContext; + this.done = done; + this.startTimeMs = startTimeMs; + } + + public long getStartTimeMs() { + return startTimeMs; + } + + public long getIndex() { + return index; + } + + public void setIndex(long index) { + this.index = index; + } + + public Bytes getRequestContext() { + return requestContext; + } + + public ReadIndexClosure getDone() { + return done; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexStatus.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexStatus.java new file mode 100644 index 0000000..747c51c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/ReadIndexStatus.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.util.List; + +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; + +/** + * ReadIndex requests statuses. + * @author dennis + * + */ +public class ReadIndexStatus { + + private final ReadIndexRequest request; // raw request + private final List states; // read index requests in batch. + private final long index; // committed log index. + + public ReadIndexStatus(List states, ReadIndexRequest request, long index) { + super(); + this.index = index; + this.request = request; + this.states = states; + } + + public boolean isApplied(long appliedIndex) { + return appliedIndex >= this.index; + } + + public boolean isOverMaxReadIndexLag(long applyIndex, int maxReadIndexLag) { + if (maxReadIndexLag < 0) { + return false; + } + return this.index - applyIndex > maxReadIndexLag; + } + + public long getIndex() { + return index; + } + + public ReadIndexRequest getRequest() { + return request; + } + + public List getStates() { + return states; + } + +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Task.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Task.java new file mode 100644 index 0000000..7681d2e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/Task.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.closure.JoinableClosure; +import com.alipay.sofa.jraft.util.Requires; + +/** + * Basic message structure of jraft, contains: + *
    + *
  • data: associated task data
    • + *
  • done: task closure, called when the data is successfully committed to the raft group.
    • + *
  • expectedTerm: Reject this task if expectedTerm doesn't match the current term of this Node if the value is not -1, default is -1.
    • + *
+ * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-13 3:08:12 PM + */ +public class Task implements Serializable { + + private static final long serialVersionUID = 2971309899898274575L; + + /** Associated task data*/ + private ByteBuffer data = LogEntry.EMPTY_DATA; + /** task closure, called when the data is successfully committed to the raft group or failures happen.*/ + private Closure done; + /** Reject this task if expectedTerm doesn't match the current term of this Node if the value is not -1, default is -1.*/ + private long expectedTerm = -1; + + public Task() { + super(); + } + + /** + * Creates a task with data/done. + */ + public Task(final ByteBuffer data, final Closure done) { + super(); + this.data = data; + this.done = done; + } + + /** + * Creates a task with data/done/expectedTerm. + */ + public Task(final ByteBuffer data, final Closure done, final long expectedTerm) { + super(); + this.data = data; + this.done = done; + this.expectedTerm = expectedTerm; + } + + public ByteBuffer getData() { + return this.data; + } + + public void setData(final ByteBuffer data) { + Requires.requireNonNull(data, "data should not be null, you can use LogEntry.EMPTY_DATA instead."); + this.data = data; + } + + public Closure getDone() { + return this.done; + } + + public void setDone(final Closure done) { + this.done = done; + } + + public long getExpectedTerm() { + return this.expectedTerm; + } + + public void setExpectedTerm(final long expectedTerm) { + this.expectedTerm = expectedTerm; + } + + /** + * Waiting for the task to complete, to note that throughput may be reduced, + * which is generally not recommended. + * + * @return done closure + * @throws InterruptedException if the current thread is interrupted while waiting + * @since 1.3.1 + */ + public Closure join() throws InterruptedException { + final JoinableClosure joinable = castToJoinalbe(this.done); + joinable.join(); + return joinable.getClosure(); + } + + /** + * Waiting for the task to complete with a timeout millis, to note that throughput + * may be reduced, which is generally not recommended. + * + * @param timeoutMillis the maximum millis to wait + * @return done closure + * @throws InterruptedException if the current thread is interrupted while waiting + * @throws TimeoutException if timeout + * @since 1.3.1 + */ + public Closure join(final long timeoutMillis) throws InterruptedException, TimeoutException { + final JoinableClosure joinable = castToJoinalbe(this.done); + joinable.join(timeoutMillis); + return joinable.getClosure(); + } + + /** + * Waiting for all tasks to complete. + * + * @param tasks task list + * @return the closure list in tasks + * @throws InterruptedException if the current thread is interrupted while waiting + * @since 1.3.1 + */ + public static List joinAll(final List tasks) throws InterruptedException { + final List closures = new ArrayList<>(tasks.size()); + for (final Task t : tasks) { + closures.add(t.join()); + } + return closures; + } + + /** + * Waiting for all tasks to complete with a timeout millis. + * + * @param tasks task list + * @param timeoutMillis the maximum millis to wait + * @return the closure list in the tasks + * @throws InterruptedException if the current thread is interrupted while waiting + * @throws TimeoutException if timeout + * @since 1.3.1 + */ + public static List joinAll(final List tasks, long timeoutMillis) throws InterruptedException, + TimeoutException { + final List closures = new ArrayList<>(tasks.size()); + for (final Task t : tasks) { + final long start = System.nanoTime(); + closures.add(t.join(timeoutMillis)); + timeoutMillis -= TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + if (timeoutMillis <= 0) { + throw new TimeoutException("joined timeout"); + } + } + return closures; + } + + private static JoinableClosure castToJoinalbe(final Closure closure) { + if (closure instanceof JoinableClosure) { + return (JoinableClosure) closure; + } + throw new UnsupportedOperationException("Unsupported join"); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/UserLog.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/UserLog.java new file mode 100644 index 0000000..eef6c23 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/UserLog.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity; + +import java.nio.ByteBuffer; + +/** + * User log entry. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 10:03:28 AM + */ +public class UserLog { + + /** log index*/ + private long index; + /** log data*/ + private ByteBuffer data; + + public UserLog(long index, ByteBuffer data) { + super(); + this.index = index; + this.data = data; + } + + public long getIndex() { + return this.index; + } + + public void setIndex(long index) { + this.index = index; + } + + public ByteBuffer getData() { + return this.data; + } + + public void setData(ByteBuffer data) { + this.data = data; + } + + public void reset() { + this.data.clear(); + this.index = 0; + } + + @Override + public String toString() { + return "UserLog [index=" + this.index + ", data=" + this.data + "]"; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/AutoDetectDecoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/AutoDetectDecoder.java new file mode 100644 index 0000000..68b6fd7 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/AutoDetectDecoder.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec; + +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.codec.v1.V1Decoder; +import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory; +import com.alipay.sofa.jraft.entity.codec.v2.V2Decoder; + +/** + * Decoder that supports both v1 and v2 log entry codec protocol. + * @author boyan(boyan@antfin.com) + * + */ +public class AutoDetectDecoder implements LogEntryDecoder { + + private AutoDetectDecoder() { + + } + + public static final AutoDetectDecoder INSTANCE = new AutoDetectDecoder(); + + @Override + public LogEntry decode(final byte[] bs) { + if (bs == null || bs.length < 1) { + return null; + } + + if (bs[0] == LogEntryV2CodecFactory.MAGIC_BYTES[0]) { + return V2Decoder.INSTANCE.decode(bs); + } else { + return V1Decoder.INSTANCE.decode(bs); + } + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/DefaultLogEntryCodecFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/DefaultLogEntryCodecFactory.java new file mode 100644 index 0000000..55a8883 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/DefaultLogEntryCodecFactory.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec; + +import com.alipay.sofa.jraft.entity.LogEntry; + +/** + * Default log entry codec factory + * @author boyan(boyan@antfin.com) + * + */ +public class DefaultLogEntryCodecFactory implements LogEntryCodecFactory { + + private DefaultLogEntryCodecFactory() { + } + + private static DefaultLogEntryCodecFactory INSTANCE = new DefaultLogEntryCodecFactory(); + + /** + * Returns a singleton instance of DefaultLogEntryCodecFactory. + * @return a singleton instance + */ + public static DefaultLogEntryCodecFactory getInstance() { + return INSTANCE; + } + + @SuppressWarnings("deprecation") + private static LogEntryEncoder ENCODER = LogEntry::encode; + + @SuppressWarnings("deprecation") + private static LogEntryDecoder DECODER = bs -> { + final LogEntry log = new LogEntry(); + if (log.decode(bs)) { + return log; + } + return null; + }; + + @Override + public LogEntryEncoder encoder() { + return ENCODER; + } + + @Override + public LogEntryDecoder decoder() { + return DECODER; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecFactory.java new file mode 100644 index 0000000..5e9effb --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecFactory.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec; + +/** + * Log entry codec factory to create encoder/decoder for LogEntry. + * + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + */ +public interface LogEntryCodecFactory { + /** + * Returns a log entry encoder. + * @return encoder instance + */ + LogEntryEncoder encoder(); + + /** + * Returns a log entry decoder. + * @return encoder instance + */ + LogEntryDecoder decoder(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryDecoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryDecoder.java new file mode 100644 index 0000000..8c8c579 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryDecoder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec; + +import com.alipay.sofa.jraft.entity.LogEntry; + +/** + * Log entry decoder + * + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + */ +public interface LogEntryDecoder { + /** + * Decode a log entry from byte array, + * return null when fail to decode. + * @param bs + * @return + */ + LogEntry decode(byte[] bs); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryEncoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryEncoder.java new file mode 100644 index 0000000..08eea4d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/LogEntryEncoder.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec; + +import com.alipay.sofa.jraft.entity.LogEntry; + +/** + * Log entry encoder + * + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + */ +public interface LogEntryEncoder { + + /** + * Encode a log entry into a byte array. + * @param log log entry + * @return encoded byte array + */ + byte[] encode(LogEntry log); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactory.java new file mode 100644 index 0000000..32a362d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactory.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v1; + +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; + +/** + * Old V1 log entry codec implementation. + * @author boyan(boyan@antfin.com) + * + */ +@Deprecated +public class LogEntryV1CodecFactory implements LogEntryCodecFactory { + + //"Beeep boop beep beep boop beeeeeep" -BB8 + public static final byte MAGIC = (byte) 0xB8; + + private LogEntryV1CodecFactory() { + } + + private static final LogEntryV1CodecFactory INSTANCE = new LogEntryV1CodecFactory(); + + /** + * Returns a singleton instance of DefaultLogEntryCodecFactory. + * @return a singleton instance + */ + public static LogEntryV1CodecFactory getInstance() { + return INSTANCE; + } + + @Override + public LogEntryEncoder encoder() { + return V1Encoder.INSTANCE; + } + + @Override + public LogEntryDecoder decoder() { + return V1Decoder.INSTANCE; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Decoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Decoder.java new file mode 100644 index 0000000..ff811b2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Decoder.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v1; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import com.alipay.sofa.jraft.entity.EnumOutter; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.alipay.sofa.jraft.util.Bits; + +/** + * V1 log entry decoder + * @author boyan(boyan@antfin.com) + * + */ +@Deprecated +public final class V1Decoder implements LogEntryDecoder { + + private V1Decoder() { + } + + public static final V1Decoder INSTANCE = new V1Decoder(); + + @Override + public LogEntry decode(final byte[] content) { + if (content == null || content.length == 0) { + return null; + } + if (content[0] != LogEntryV1CodecFactory.MAGIC) { + // Corrupted log + return null; + } + LogEntry log = new LogEntry(); + decode(log, content); + + return log; + } + + public void decode(final LogEntry log, final byte[] content) { + // 1-5 type + final int iType = Bits.getInt(content, 1); + log.setType(EnumOutter.EntryType.forNumber(iType)); + // 5-13 index + // 13-21 term + final long index = Bits.getLong(content, 5); + final long term = Bits.getLong(content, 13); + log.setId(new LogId(index, term)); + // 21-25 peer count + int peerCount = Bits.getInt(content, 21); + // peers + int pos = 25; + if (peerCount > 0) { + List peers = new ArrayList<>(peerCount); + while (peerCount-- > 0) { + final short len = Bits.getShort(content, pos); + final byte[] bs = new byte[len]; + System.arraycopy(content, pos + 2, bs, 0, len); + // peer len (short in 2 bytes) + // peer str + pos += 2 + len; + final PeerId peer = new PeerId(); + peer.parse(AsciiStringUtil.unsafeDecode(bs)); + peers.add(peer); + } + log.setPeers(peers); + } + // old peers + int oldPeerCount = Bits.getInt(content, pos); + pos += 4; + if (oldPeerCount > 0) { + List oldPeers = new ArrayList<>(oldPeerCount); + while (oldPeerCount-- > 0) { + final short len = Bits.getShort(content, pos); + final byte[] bs = new byte[len]; + System.arraycopy(content, pos + 2, bs, 0, len); + // peer len (short in 2 bytes) + // peer str + pos += 2 + len; + final PeerId peer = new PeerId(); + peer.parse(AsciiStringUtil.unsafeDecode(bs)); + oldPeers.add(peer); + } + log.setOldPeers(oldPeers); + } + + // data + if (content.length > pos) { + final int len = content.length - pos; + ByteBuffer data = ByteBuffer.allocate(len); + data.put(content, pos, len); + data.flip(); + log.setData(data); + } + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Encoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Encoder.java new file mode 100644 index 0000000..2df92a9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v1/V1Encoder.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v1; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import com.alipay.sofa.jraft.entity.EnumOutter.EntryType; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.alipay.sofa.jraft.util.Bits; + +/** + * V1 log entry encoder + * @author boyan(boyan@antfin.com) + * + */ +@Deprecated +public final class V1Encoder implements LogEntryEncoder { + + private V1Encoder() { + } + + public static final LogEntryEncoder INSTANCE = new V1Encoder(); + + @Override + public byte[] encode(final LogEntry log) { + if (log.hasLearners()) { + throw new IllegalArgumentException("V1 log entry encoder doesn't support learners"); + } + EntryType type = log.getType(); + LogId id = log.getId(); + List peers = log.getPeers(); + List oldPeers = log.getOldPeers(); + ByteBuffer data = log.getData(); + + // magic number 1 byte + int totalLen = 1; + final int iType = type.getNumber(); + final long index = id.getIndex(); + final long term = id.getTerm(); + // type(4) + index(8) + term(8) + totalLen += 4 + 8 + 8; + int peerCount = 0; + // peer count + totalLen += 4; + final List peerStrs = new ArrayList<>(peerCount); + if (peers != null) { + peerCount = peers.size(); + for (final PeerId peer : peers) { + final String peerStr = peer.toString(); + // peer len (short in 2 bytes) + // peer str + totalLen += 2 + peerStr.length(); + peerStrs.add(peerStr); + } + } + int oldPeerCount = 0; + // old peer count + totalLen += 4; + final List oldPeerStrs = new ArrayList<>(oldPeerCount); + if (oldPeers != null) { + oldPeerCount = oldPeers.size(); + for (final PeerId peer : oldPeers) { + final String peerStr = peer.toString(); + // peer len (short in 2 bytes) + // peer str + totalLen += 2 + peerStr.length(); + oldPeerStrs.add(peerStr); + } + } + + final int bodyLen = data != null ? data.remaining() : 0; + totalLen += bodyLen; + + final byte[] content = new byte[totalLen]; + // {0} magic + content[0] = LogEntryV1CodecFactory.MAGIC; + // 1-5 type + Bits.putInt(content, 1, iType); + // 5-13 index + Bits.putLong(content, 5, index); + // 13-21 term + Bits.putLong(content, 13, term); + // peers + // 21-25 peer count + Bits.putInt(content, 21, peerCount); + int pos = 25; + for (final String peerStr : peerStrs) { + final byte[] ps = AsciiStringUtil.unsafeEncode(peerStr); + Bits.putShort(content, pos, (short) peerStr.length()); + System.arraycopy(ps, 0, content, pos + 2, ps.length); + pos += 2 + ps.length; + } + // old peers + // old peers count + Bits.putInt(content, pos, oldPeerCount); + pos += 4; + for (final String peerStr : oldPeerStrs) { + final byte[] ps = AsciiStringUtil.unsafeEncode(peerStr); + Bits.putShort(content, pos, (short) peerStr.length()); + System.arraycopy(ps, 0, content, pos + 2, ps.length); + pos += 2 + ps.length; + } + // data + if (data != null) { + System.arraycopy(data.array(), data.position(), content, pos, data.remaining()); + } + + return content; + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactory.java new file mode 100644 index 0000000..7a2763f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactory.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v2; + +import com.alipay.sofa.jraft.entity.codec.AutoDetectDecoder; +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; + +/** + * V2(Now) log entry codec implementation, header format: + * + * 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-++-+-+-+ + * |Magic|Version|Reserved| + * +-+-+-+-+-+-+-+-++-+-+-+ + * + * @author boyan(boyan@antfin.com) + * @since 1.2.6 + * + */ +public class LogEntryV2CodecFactory implements LogEntryCodecFactory { + + private static final LogEntryV2CodecFactory INSTANCE = new LogEntryV2CodecFactory(); + + public static LogEntryV2CodecFactory getInstance() { + return INSTANCE; + } + + // BB-8 and R2D2 are good friends. + public static final byte[] MAGIC_BYTES = new byte[] { (byte) 0xBB, (byte) 0xD2 }; + // Codec version + public static final byte VERSION = 1; + + public static final byte[] RESERVED = new byte[3]; + + public static final int HEADER_SIZE = MAGIC_BYTES.length + 1 + RESERVED.length; + + @Override + public LogEntryEncoder encoder() { + return V2Encoder.INSTANCE; + } + + @Override + public LogEntryDecoder decoder() { + return AutoDetectDecoder.INSTANCE; + } + + private LogEntryV2CodecFactory() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogOutter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogOutter.java new file mode 100644 index 0000000..822c457 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/LogOutter.java @@ -0,0 +1,1604 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: log.proto + +package com.alipay.sofa.jraft.entity.codec.v2; + +public final class LogOutter { + private LogOutter() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public interface PBLogEntryOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.PBLogEntry) + com.google.protobuf.MessageOrBuilder { + + /** + * required .jraft.EntryType type = 1; + */ + boolean hasType(); + + /** + * required .jraft.EntryType type = 1; + */ + com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType(); + + /** + * required int64 term = 2; + */ + boolean hasTerm(); + + /** + * required int64 term = 2; + */ + long getTerm(); + + /** + * required int64 index = 3; + */ + boolean hasIndex(); + + /** + * required int64 index = 3; + */ + long getIndex(); + + /** + * repeated bytes peers = 4; + */ + java.util.List getPeersList(); + + /** + * repeated bytes peers = 4; + */ + int getPeersCount(); + + /** + * repeated bytes peers = 4; + */ + com.google.protobuf.ByteString getPeers(int index); + + /** + * repeated bytes old_peers = 5; + */ + java.util.List getOldPeersList(); + + /** + * repeated bytes old_peers = 5; + */ + int getOldPeersCount(); + + /** + * repeated bytes old_peers = 5; + */ + com.google.protobuf.ByteString getOldPeers(int index); + + /** + * required bytes data = 6; + */ + boolean hasData(); + + /** + * required bytes data = 6; + */ + com.google.protobuf.ByteString getData(); + + /** + * optional int64 checksum = 7; + */ + boolean hasChecksum(); + + /** + * optional int64 checksum = 7; + */ + long getChecksum(); + + /** + * repeated bytes learners = 8; + */ + java.util.List getLearnersList(); + + /** + * repeated bytes learners = 8; + */ + int getLearnersCount(); + + /** + * repeated bytes learners = 8; + */ + com.google.protobuf.ByteString getLearners(int index); + + /** + * repeated bytes old_learners = 9; + */ + java.util.List getOldLearnersList(); + + /** + * repeated bytes old_learners = 9; + */ + int getOldLearnersCount(); + + /** + * repeated bytes old_learners = 9; + */ + com.google.protobuf.ByteString getOldLearners(int index); + } + + /** + * Protobuf type {@code jraft.PBLogEntry} + */ + public static final class PBLogEntry extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.PBLogEntry) + PBLogEntryOrBuilder { + private static final long serialVersionUID = 0L; + + // Use PBLogEntry.newBuilder() to construct. + private PBLogEntry(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private PBLogEntry() { + type_ = 0; + term_ = 0L; + index_ = 0L; + peers_ = java.util.Collections.emptyList(); + oldPeers_ = java.util.Collections.emptyList(); + data_ = com.google.protobuf.ByteString.EMPTY; + checksum_ = 0L; + learners_ = java.util.Collections.emptyList(); + oldLearners_ = java.util.Collections.emptyList(); + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private PBLogEntry(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + int rawValue = input.readEnum(); + com.alipay.sofa.jraft.entity.EnumOutter.EntryType value = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(rawValue); + if (value == null) { + unknownFields.mergeVarintField(1, rawValue); + } else { + bitField0_ |= 0x00000001; + type_ = rawValue; + } + break; + } + case 16: { + bitField0_ |= 0x00000002; + term_ = input.readInt64(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + index_ = input.readInt64(); + break; + } + case 34: { + if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + peers_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000008; + } + peers_.add(input.readBytes()); + break; + } + case 42: { + if (!((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000010; + } + oldPeers_.add(input.readBytes()); + break; + } + case 50: { + bitField0_ |= 0x00000008; + data_ = input.readBytes(); + break; + } + case 56: { + bitField0_ |= 0x00000010; + checksum_ = input.readInt64(); + break; + } + case 66: { + if (!((mutable_bitField0_ & 0x00000080) == 0x00000080)) { + learners_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000080; + } + learners_.add(input.readBytes()); + break; + } + case 74: { + if (!((mutable_bitField0_ & 0x00000100) == 0x00000100)) { + oldLearners_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000100; + } + oldLearners_.add(input.readBytes()); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + peers_ = java.util.Collections.unmodifiableList(peers_); + } + if (((mutable_bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = java.util.Collections.unmodifiableList(oldPeers_); + } + if (((mutable_bitField0_ & 0x00000080) == 0x00000080)) { + learners_ = java.util.Collections.unmodifiableList(learners_); + } + if (((mutable_bitField0_ & 0x00000100) == 0x00000100)) { + oldLearners_ = java.util.Collections.unmodifiableList(oldLearners_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.internal_static_jraft_PBLogEntry_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.internal_static_jraft_PBLogEntry_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.class, + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.Builder.class); + } + + private int bitField0_; + public static final int TYPE_FIELD_NUMBER = 1; + private int type_; + + /** + * required .jraft.EntryType type = 1; + */ + public boolean hasType() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required .jraft.EntryType type = 1; + */ + public com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType() { + com.alipay.sofa.jraft.entity.EnumOutter.EntryType result = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(type_); + return result == null ? com.alipay.sofa.jraft.entity.EnumOutter.EntryType.ENTRY_TYPE_UNKNOWN : result; + } + + public static final int TERM_FIELD_NUMBER = 2; + private long term_; + + /** + * required int64 term = 2; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required int64 term = 2; + */ + public long getTerm() { + return term_; + } + + public static final int INDEX_FIELD_NUMBER = 3; + private long index_; + + /** + * required int64 index = 3; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required int64 index = 3; + */ + public long getIndex() { + return index_; + } + + public static final int PEERS_FIELD_NUMBER = 4; + private java.util.List peers_; + + /** + * repeated bytes peers = 4; + */ + public java.util.List getPeersList() { + return peers_; + } + + /** + * repeated bytes peers = 4; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated bytes peers = 4; + */ + public com.google.protobuf.ByteString getPeers(int index) { + return peers_.get(index); + } + + public static final int OLD_PEERS_FIELD_NUMBER = 5; + private java.util.List oldPeers_; + + /** + * repeated bytes old_peers = 5; + */ + public java.util.List getOldPeersList() { + return oldPeers_; + } + + /** + * repeated bytes old_peers = 5; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated bytes old_peers = 5; + */ + public com.google.protobuf.ByteString getOldPeers(int index) { + return oldPeers_.get(index); + } + + public static final int DATA_FIELD_NUMBER = 6; + private com.google.protobuf.ByteString data_; + + /** + * required bytes data = 6; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required bytes data = 6; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + public static final int CHECKSUM_FIELD_NUMBER = 7; + private long checksum_; + + /** + * optional int64 checksum = 7; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * optional int64 checksum = 7; + */ + public long getChecksum() { + return checksum_; + } + + public static final int LEARNERS_FIELD_NUMBER = 8; + private java.util.List learners_; + + /** + * repeated bytes learners = 8; + */ + public java.util.List getLearnersList() { + return learners_; + } + + /** + * repeated bytes learners = 8; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated bytes learners = 8; + */ + public com.google.protobuf.ByteString getLearners(int index) { + return learners_.get(index); + } + + public static final int OLD_LEARNERS_FIELD_NUMBER = 9; + private java.util.List oldLearners_; + + /** + * repeated bytes old_learners = 9; + */ + public java.util.List getOldLearnersList() { + return oldLearners_; + } + + /** + * repeated bytes old_learners = 9; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated bytes old_learners = 9; + */ + public com.google.protobuf.ByteString getOldLearners(int index) { + return oldLearners_.get(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasType()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasIndex()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasData()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeEnum(1, type_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeInt64(2, term_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, index_); + } + for (int i = 0; i < peers_.size(); i++) { + output.writeBytes(4, peers_.get(i)); + } + for (int i = 0; i < oldPeers_.size(); i++) { + output.writeBytes(5, oldPeers_.get(i)); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeBytes(6, data_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(7, checksum_); + } + for (int i = 0; i < learners_.size(); i++) { + output.writeBytes(8, learners_.get(i)); + } + for (int i = 0; i < oldLearners_.size(); i++) { + output.writeBytes(9, oldLearners_.get(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeEnumSize(1, type_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(2, term_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(3, index_); + } + { + int dataSize = 0; + for (int i = 0; i < peers_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(peers_.get(i)); + } + size += dataSize; + size += 1 * getPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(oldPeers_.get(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeBytesSize(6, data_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(7, checksum_); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(learners_.get(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < oldLearners_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(oldLearners_.get(i)); + } + size += dataSize; + size += 1 * getOldLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry other = (com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry) obj; + + boolean result = true; + result = result && (hasType() == other.hasType()); + if (hasType()) { + result = result && type_ == other.type_; + } + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasIndex() == other.hasIndex()); + if (hasIndex()) { + result = result && (getIndex() == other.getIndex()); + } + result = result && getPeersList().equals(other.getPeersList()); + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && (hasData() == other.hasData()); + if (hasData()) { + result = result && getData().equals(other.getData()); + } + result = result && (hasChecksum() == other.hasChecksum()); + if (hasChecksum()) { + result = result && (getChecksum() == other.getChecksum()); + } + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && getOldLearnersList().equals(other.getOldLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasType()) { + hash = (37 * hash) + TYPE_FIELD_NUMBER; + hash = (53 * hash) + type_; + } + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasIndex()) { + hash = (37 * hash) + INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getIndex()); + } + if (getPeersCount() > 0) { + hash = (37 * hash) + PEERS_FIELD_NUMBER; + hash = (53 * hash) + getPeersList().hashCode(); + } + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (hasData()) { + hash = (37 * hash) + DATA_FIELD_NUMBER; + hash = (53 * hash) + getData().hashCode(); + } + if (hasChecksum()) { + hash = (37 * hash) + CHECKSUM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getChecksum()); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + if (getOldLearnersCount() > 0) { + hash = (37 * hash) + OLD_LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getOldLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.PBLogEntry} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.PBLogEntry) + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntryOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.internal_static_jraft_PBLogEntry_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.internal_static_jraft_PBLogEntry_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.class, + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + type_ = 0; + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000002); + index_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + peers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + oldPeers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + data_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000020); + checksum_ = 0L; + bitField0_ = (bitField0_ & ~0x00000040); + learners_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000080); + oldLearners_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000100); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.internal_static_jraft_PBLogEntry_descriptor; + } + + public com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry getDefaultInstanceForType() { + return com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry build() { + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry buildPartial() { + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry result = new com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.type_ = type_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.index_ = index_; + if (((bitField0_ & 0x00000008) == 0x00000008)) { + peers_ = java.util.Collections.unmodifiableList(peers_); + bitField0_ = (bitField0_ & ~0x00000008); + } + result.peers_ = peers_; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = java.util.Collections.unmodifiableList(oldPeers_); + bitField0_ = (bitField0_ & ~0x00000010); + } + result.oldPeers_ = oldPeers_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000008; + } + result.data_ = data_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000010; + } + result.checksum_ = checksum_; + if (((bitField0_ & 0x00000080) == 0x00000080)) { + learners_ = java.util.Collections.unmodifiableList(learners_); + bitField0_ = (bitField0_ & ~0x00000080); + } + result.learners_ = learners_; + if (((bitField0_ & 0x00000100) == 0x00000100)) { + oldLearners_ = java.util.Collections.unmodifiableList(oldLearners_); + bitField0_ = (bitField0_ & ~0x00000100); + } + result.oldLearners_ = oldLearners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry) { + return mergeFrom((com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry other) { + if (other == com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry.getDefaultInstance()) + return this; + if (other.hasType()) { + setType(other.getType()); + } + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasIndex()) { + setIndex(other.getIndex()); + } + if (!other.peers_.isEmpty()) { + if (peers_.isEmpty()) { + peers_ = other.peers_; + bitField0_ = (bitField0_ & ~0x00000008); + } else { + ensurePeersIsMutable(); + peers_.addAll(other.peers_); + } + onChanged(); + } + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000010); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (other.hasData()) { + setData(other.getData()); + } + if (other.hasChecksum()) { + setChecksum(other.getChecksum()); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000080); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + if (!other.oldLearners_.isEmpty()) { + if (oldLearners_.isEmpty()) { + oldLearners_ = other.oldLearners_; + bitField0_ = (bitField0_ & ~0x00000100); + } else { + ensureOldLearnersIsMutable(); + oldLearners_.addAll(other.oldLearners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasType()) { + return false; + } + if (!hasTerm()) { + return false; + } + if (!hasIndex()) { + return false; + } + if (!hasData()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private int type_ = 0; + + /** + * required .jraft.EntryType type = 1; + */ + public boolean hasType() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required .jraft.EntryType type = 1; + */ + public com.alipay.sofa.jraft.entity.EnumOutter.EntryType getType() { + com.alipay.sofa.jraft.entity.EnumOutter.EntryType result = com.alipay.sofa.jraft.entity.EnumOutter.EntryType + .valueOf(type_); + return result == null ? com.alipay.sofa.jraft.entity.EnumOutter.EntryType.ENTRY_TYPE_UNKNOWN : result; + } + + /** + * required .jraft.EntryType type = 1; + */ + public Builder setType(com.alipay.sofa.jraft.entity.EnumOutter.EntryType value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + type_ = value.getNumber(); + onChanged(); + return this; + } + + /** + * required .jraft.EntryType type = 1; + */ + public Builder clearType() { + bitField0_ = (bitField0_ & ~0x00000001); + type_ = 0; + onChanged(); + return this; + } + + private long term_; + + /** + * required int64 term = 2; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required int64 term = 2; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 2; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000002; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 2; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000002); + term_ = 0L; + onChanged(); + return this; + } + + private long index_; + + /** + * required int64 index = 3; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required int64 index = 3; + */ + public long getIndex() { + return index_; + } + + /** + * required int64 index = 3; + */ + public Builder setIndex(long value) { + bitField0_ |= 0x00000004; + index_ = value; + onChanged(); + return this; + } + + /** + * required int64 index = 3; + */ + public Builder clearIndex() { + bitField0_ = (bitField0_ & ~0x00000004); + index_ = 0L; + onChanged(); + return this; + } + + private java.util.List peers_ = java.util.Collections.emptyList(); + + private void ensurePeersIsMutable() { + if (!((bitField0_ & 0x00000008) == 0x00000008)) { + peers_ = new java.util.ArrayList(peers_); + bitField0_ |= 0x00000008; + } + } + + /** + * repeated bytes peers = 4; + */ + public java.util.List getPeersList() { + return java.util.Collections.unmodifiableList(peers_); + } + + /** + * repeated bytes peers = 4; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated bytes peers = 4; + */ + public com.google.protobuf.ByteString getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated bytes peers = 4; + */ + public Builder setPeers(int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated bytes peers = 4; + */ + public Builder addPeers(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + /** + * repeated bytes peers = 4; + */ + public Builder addAllPeers(java.lang.Iterable values) { + ensurePeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, peers_); + onChanged(); + return this; + } + + /** + * repeated bytes peers = 4; + */ + public Builder clearPeers() { + peers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000008); + onChanged(); + return this; + } + + private java.util.List oldPeers_ = java.util.Collections.emptyList(); + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000010) == 0x00000010)) { + oldPeers_ = new java.util.ArrayList(oldPeers_); + bitField0_ |= 0x00000010; + } + } + + /** + * repeated bytes old_peers = 5; + */ + public java.util.List getOldPeersList() { + return java.util.Collections.unmodifiableList(oldPeers_); + } + + /** + * repeated bytes old_peers = 5; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated bytes old_peers = 5; + */ + public com.google.protobuf.ByteString getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated bytes old_peers = 5; + */ + public Builder setOldPeers(int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated bytes old_peers = 5; + */ + public Builder addOldPeers(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated bytes old_peers = 5; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated bytes old_peers = 5; + */ + public Builder clearOldPeers() { + oldPeers_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000010); + onChanged(); + return this; + } + + private com.google.protobuf.ByteString data_ = com.google.protobuf.ByteString.EMPTY; + + /** + * required bytes data = 6; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required bytes data = 6; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + /** + * required bytes data = 6; + */ + public Builder setData(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + data_ = value; + onChanged(); + return this; + } + + /** + * required bytes data = 6; + */ + public Builder clearData() { + bitField0_ = (bitField0_ & ~0x00000020); + data_ = getDefaultInstance().getData(); + onChanged(); + return this; + } + + private long checksum_; + + /** + * optional int64 checksum = 7; + */ + public boolean hasChecksum() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + + /** + * optional int64 checksum = 7; + */ + public long getChecksum() { + return checksum_; + } + + /** + * optional int64 checksum = 7; + */ + public Builder setChecksum(long value) { + bitField0_ |= 0x00000040; + checksum_ = value; + onChanged(); + return this; + } + + /** + * optional int64 checksum = 7; + */ + public Builder clearChecksum() { + bitField0_ = (bitField0_ & ~0x00000040); + checksum_ = 0L; + onChanged(); + return this; + } + + private java.util.List learners_ = java.util.Collections.emptyList(); + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000080) == 0x00000080)) { + learners_ = new java.util.ArrayList(learners_); + bitField0_ |= 0x00000080; + } + } + + /** + * repeated bytes learners = 8; + */ + public java.util.List getLearnersList() { + return java.util.Collections.unmodifiableList(learners_); + } + + /** + * repeated bytes learners = 8; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated bytes learners = 8; + */ + public com.google.protobuf.ByteString getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated bytes learners = 8; + */ + public Builder setLearners(int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated bytes learners = 8; + */ + public Builder addLearners(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated bytes learners = 8; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated bytes learners = 8; + */ + public Builder clearLearners() { + learners_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000080); + onChanged(); + return this; + } + + private java.util.List oldLearners_ = java.util.Collections.emptyList(); + + private void ensureOldLearnersIsMutable() { + if (!((bitField0_ & 0x00000100) == 0x00000100)) { + oldLearners_ = new java.util.ArrayList(oldLearners_); + bitField0_ |= 0x00000100; + } + } + + /** + * repeated bytes old_learners = 9; + */ + public java.util.List getOldLearnersList() { + return java.util.Collections.unmodifiableList(oldLearners_); + } + + /** + * repeated bytes old_learners = 9; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated bytes old_learners = 9; + */ + public com.google.protobuf.ByteString getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated bytes old_learners = 9; + */ + public Builder setOldLearners(int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated bytes old_learners = 9; + */ + public Builder addOldLearners(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + /** + * repeated bytes old_learners = 9; + */ + public Builder addAllOldLearners(java.lang.Iterable values) { + ensureOldLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldLearners_); + onChanged(); + return this; + } + + /** + * repeated bytes old_learners = 9; + */ + public Builder clearOldLearners() { + oldLearners_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000100); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.PBLogEntry) + } + + // @@protoc_insertion_point(class_scope:jraft.PBLogEntry) + private static final com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry(); + } + + public static com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public PBLogEntry parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new PBLogEntry(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_PBLogEntry_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_PBLogEntry_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\tlog.proto\022\005jraft\032\nenum.proto\"\263\001\n\nPBLog" + + "Entry\022\036\n\004type\030\001 \002(\0162\020.jraft.EntryType\022\014\n" + + "\004term\030\002 \002(\003\022\r\n\005index\030\003 \002(\003\022\r\n\005peers\030\004 \003(" + + "\014\022\021\n\told_peers\030\005 \003(\014\022\014\n\004data\030\006 \002(\014\022\020\n\010ch" + + "ecksum\030\007 \001(\003\022\020\n\010learners\030\010 \003(\014\022\024\n\014old_le" + + "arners\030\t \003(\014B2\n%com.alipay.sofa.jraft.en" + + "tity.codec.v2B\tLogOutter" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { com.alipay.sofa.jraft.entity.EnumOutter + .getDescriptor(), }, assigner); + internal_static_jraft_PBLogEntry_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_PBLogEntry_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_PBLogEntry_descriptor, new java.lang.String[] { "Type", "Term", "Index", "Peers", + "OldPeers", "Data", "Checksum", "Learners", "OldLearners", }); + com.alipay.sofa.jraft.entity.EnumOutter.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Decoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Decoder.java new file mode 100644 index 0000000..edc98c2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Decoder.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v2; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.ZeroByteStringHelper; + +/** + * V2 log entry decoder based on protobuf, see src/main/resources/log.proto + * + * @author boyan(boyan@antfin.com) + */ +public class V2Decoder implements LogEntryDecoder { + + private static final Logger LOG = LoggerFactory.getLogger(V2Decoder.class); + + public static final V2Decoder INSTANCE = new V2Decoder(); + + @Override + public LogEntry decode(final byte[] bs) { + if (bs == null || bs.length < LogEntryV2CodecFactory.HEADER_SIZE) { + return null; + } + + int i = 0; + for (; i < LogEntryV2CodecFactory.MAGIC_BYTES.length; i++) { + if (bs[i] != LogEntryV2CodecFactory.MAGIC_BYTES[i]) { + return null; + } + } + + if (bs[i++] != LogEntryV2CodecFactory.VERSION) { + return null; + } + // Ignored reserved + i += LogEntryV2CodecFactory.RESERVED.length; + try { + final PBLogEntry entry = PBLogEntry.parseFrom(ZeroByteStringHelper.wrap(bs, i, bs.length - i)); + + final LogEntry log = new LogEntry(); + log.setType(entry.getType()); + log.getId().setIndex(entry.getIndex()); + log.getId().setTerm(entry.getTerm()); + + if (entry.hasChecksum()) { + log.setChecksum(entry.getChecksum()); + } + if (entry.getPeersCount() > 0) { + final List peers = new ArrayList<>(entry.getPeersCount()); + for (final ByteString bstring : entry.getPeersList()) { + peers.add(JRaftUtils.getPeerId(AsciiStringUtil.unsafeDecode(bstring))); + } + log.setPeers(peers); + } + if (entry.getOldPeersCount() > 0) { + final List peers = new ArrayList<>(entry.getOldPeersCount()); + for (final ByteString bstring : entry.getOldPeersList()) { + peers.add(JRaftUtils.getPeerId(AsciiStringUtil.unsafeDecode(bstring))); + } + log.setOldPeers(peers); + } + + if (entry.getLearnersCount() > 0) { + final List peers = new ArrayList<>(entry.getLearnersCount()); + for (final ByteString bstring : entry.getLearnersList()) { + peers.add(JRaftUtils.getPeerId(AsciiStringUtil.unsafeDecode(bstring))); + } + log.setLearners(peers); + } + + if (entry.getOldLearnersCount() > 0) { + final List peers = new ArrayList<>(entry.getOldLearnersCount()); + for (final ByteString bstring : entry.getOldLearnersList()) { + peers.add(JRaftUtils.getPeerId(AsciiStringUtil.unsafeDecode(bstring))); + } + log.setOldLearners(peers); + } + + final ByteString data = entry.getData(); + if (!data.isEmpty()) { + log.setData(ByteBuffer.wrap(ZeroByteStringHelper.getByteArray(data))); + } + + return log; + } catch (final InvalidProtocolBufferException e) { + LOG.error("Fail to decode pb log entry", e); + return null; + } + } + + private V2Decoder() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Encoder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Encoder.java new file mode 100644 index 0000000..22605a9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/entity/codec/v2/V2Encoder.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.entity.codec.v2; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; + +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; +import com.alipay.sofa.jraft.entity.codec.v2.LogOutter.PBLogEntry; +import com.alipay.sofa.jraft.error.LogEntryCorruptedException; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.alipay.sofa.jraft.util.Requires; +import com.google.protobuf.ByteString; +import com.google.protobuf.CodedOutputStream; +import com.google.protobuf.ZeroByteStringHelper; + +/** + * V2 log entry encoder based on protobuf, see src/main/resources/log.proto + * + * @author boyan(boyan@antfin.com) + */ +public class V2Encoder implements LogEntryEncoder { + + public static final V2Encoder INSTANCE = new V2Encoder(); + + private static boolean hasPeers(final Collection peers) { + return peers != null && !peers.isEmpty(); + } + + private void encodePeers(final PBLogEntry.Builder builder, final List peers) { + final int size = peers.size(); + for (int i = 0; i < size; i++) { + builder.addPeers(ZeroByteStringHelper.wrap(AsciiStringUtil.unsafeEncode(peers.get(i).toString()))); + } + } + + private void encodeOldPeers(final PBLogEntry.Builder builder, final List peers) { + final int size = peers.size(); + for (int i = 0; i < size; i++) { + builder.addOldPeers(ZeroByteStringHelper.wrap(AsciiStringUtil.unsafeEncode(peers.get(i).toString()))); + } + } + + private void encodeLearners(final PBLogEntry.Builder builder, final List learners) { + final int size = learners.size(); + for (int i = 0; i < size; i++) { + builder.addLearners(ZeroByteStringHelper.wrap(AsciiStringUtil.unsafeEncode(learners.get(i).toString()))); + } + } + + private void encodeOldLearners(final PBLogEntry.Builder builder, final List learners) { + final int size = learners.size(); + for (int i = 0; i < size; i++) { + builder.addOldLearners(ZeroByteStringHelper.wrap(AsciiStringUtil.unsafeEncode(learners.get(i).toString()))); + } + } + + @Override + public byte[] encode(final LogEntry log) { + Requires.requireNonNull(log, "Null log"); + + final LogId logId = log.getId(); + final PBLogEntry.Builder builder = PBLogEntry.newBuilder() // + .setType(log.getType()) // + .setIndex(logId.getIndex()) // + .setTerm(logId.getTerm()); + + final List peers = log.getPeers(); + if (hasPeers(peers)) { + encodePeers(builder, peers); + } + + final List oldPeers = log.getOldPeers(); + if (hasPeers(oldPeers)) { + encodeOldPeers(builder, oldPeers); + } + + final List learners = log.getLearners(); + if (hasPeers(learners)) { + encodeLearners(builder, learners); + } + final List oldLearners = log.getOldLearners(); + if (hasPeers(oldLearners)) { + encodeOldLearners(builder, oldLearners); + } + + if (log.hasChecksum()) { + builder.setChecksum(log.getChecksum()); + } + + builder.setData(log.getData() != null ? ZeroByteStringHelper.wrap(log.getData()) : ByteString.EMPTY); + + final PBLogEntry pbLogEntry = builder.build(); + final int bodyLen = pbLogEntry.getSerializedSize(); + final byte[] ret = new byte[LogEntryV2CodecFactory.HEADER_SIZE + bodyLen]; + + // write header + int i = 0; + for (; i < LogEntryV2CodecFactory.MAGIC_BYTES.length; i++) { + ret[i] = LogEntryV2CodecFactory.MAGIC_BYTES[i]; + } + ret[i++] = LogEntryV2CodecFactory.VERSION; + // avoid memory copy for only 3 bytes + for (; i < LogEntryV2CodecFactory.HEADER_SIZE; i++) { + ret[i] = LogEntryV2CodecFactory.RESERVED[i - LogEntryV2CodecFactory.MAGIC_BYTES.length - 1]; + } + + // write body + writeToByteArray(pbLogEntry, ret, i, bodyLen); + + return ret; + } + + private void writeToByteArray(final PBLogEntry pbLogEntry, final byte[] array, final int offset, final int len) { + final CodedOutputStream output = CodedOutputStream.newInstance(array, offset, len); + try { + pbLogEntry.writeTo(output); + output.checkNoSpaceLeft(); + } catch (final IOException e) { + throw new LogEntryCorruptedException( + "Serializing PBLogEntry to a byte array threw an IOException (should never happen).", e); + } + } + + private V2Encoder() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/InvokeTimeoutException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/InvokeTimeoutException.java new file mode 100644 index 0000000..68dba43 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/InvokeTimeoutException.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * @author jiachun.fjc + */ +public class InvokeTimeoutException extends RemotingException { + + private static final long serialVersionUID = -4710810309766380565L; + + public InvokeTimeoutException() { + } + + public InvokeTimeoutException(String message) { + super(message); + } + + public InvokeTimeoutException(String message, Throwable cause) { + super(message, cause); + } + + public InvokeTimeoutException(Throwable cause) { + super(cause); + } + + public InvokeTimeoutException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/JRaftException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/JRaftException.java new file mode 100644 index 0000000..a3c4ec9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/JRaftException.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * + * @author jiachun.fjc + */ +public class JRaftException extends RuntimeException { + + private static final long serialVersionUID = 0L; + + public JRaftException() { + } + + public JRaftException(String message) { + super(message); + } + + public JRaftException(String message, Throwable cause) { + super(message, cause); + } + + public JRaftException(Throwable cause) { + super(cause); + } + + public JRaftException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogEntryCorruptedException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogEntryCorruptedException.java new file mode 100644 index 0000000..ffb141f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogEntryCorruptedException.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * LogEntry corrupted exception. + * @author boyan(boyan@antfin.com) + * + */ +public class LogEntryCorruptedException extends JRaftException { + private static final long serialVersionUID = 5664520219607766929L; + + public LogEntryCorruptedException() { + super(); + + } + + public LogEntryCorruptedException(final String message, final Throwable cause, final boolean enableSuppression, + final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + + } + + public LogEntryCorruptedException(final String message, final Throwable cause) { + super(message, cause); + + } + + public LogEntryCorruptedException(final String message) { + super(message); + + } + + public LogEntryCorruptedException(final Throwable cause) { + super(cause); + + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogIndexOutOfBoundsException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogIndexOutOfBoundsException.java new file mode 100644 index 0000000..e65dc67 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogIndexOutOfBoundsException.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Log index out of bounds + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-23 3:06:38 PM + */ +public class LogIndexOutOfBoundsException extends IndexOutOfBoundsException { + + private static final long serialVersionUID = -1096992049027390572L; + + /** + * Constructs an LogIndexOutOfBoundsException with no + * detail message. + */ + public LogIndexOutOfBoundsException() { + super(); + } + + /** + * Constructs a new LogIndexOutOfBoundsException + * class with an argument indicating the illegal index. + * + * @param index the illegal index. + */ + public LogIndexOutOfBoundsException(int index) { + super("Array index out of range: " + index); + } + + /** + * Constructs an LogIndexOutOfBoundsException class + * with the specified detail message. + * + * @param s the detail message. + */ + public LogIndexOutOfBoundsException(String s) { + super(s); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogNotFoundException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogNotFoundException.java new file mode 100644 index 0000000..787cee8 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/LogNotFoundException.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Log not found exception, the log may be deleted. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-23 3:07:56 PM + */ +public class LogNotFoundException extends IllegalStateException { + + private static final long serialVersionUID = -140969527148366390L; + + public LogNotFoundException() { + super(); + } + + public LogNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public LogNotFoundException(String s) { + super(s); + } + + public LogNotFoundException(Throwable cause) { + super(cause); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/MessageClassNotFoundException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/MessageClassNotFoundException.java new file mode 100644 index 0000000..fcab3fa --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/MessageClassNotFoundException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Protobuf message class not found exception. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-19 5:17:38 PM + */ +public class MessageClassNotFoundException extends RuntimeException { + + private static final long serialVersionUID = 4684584394785943114L; + + public MessageClassNotFoundException() { + super(); + } + + public MessageClassNotFoundException(String message, Throwable cause, boolean enableSuppression, + boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public MessageClassNotFoundException(String message, Throwable cause) { + super(message, cause); + } + + public MessageClassNotFoundException(String message) { + super(message); + } + + public MessageClassNotFoundException(Throwable cause) { + super(cause); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/OverloadException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/OverloadException.java new file mode 100644 index 0000000..e72e978 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/OverloadException.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Threw when Node is overloaded. + * @author boyan(boyan@antfin.com) + * + */ +public class OverloadException extends JRaftException { + + /** + * + */ + private static final long serialVersionUID = -5505054326197103575L; + + public OverloadException() { + super(); + } + + public OverloadException(final String message, final Throwable cause, final boolean enableSuppression, + final boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public OverloadException(final String message, final Throwable cause) { + super(message, cause); + } + + public OverloadException(final String message) { + super(message); + } + + public OverloadException(final Throwable cause) { + super(cause); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftError.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftError.java new file mode 100644 index 0000000..017e480 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftError.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +import java.util.HashMap; +import java.util.Map; + +/** + * Raft error code. + */ +public enum RaftError { + + /** + * Unknown error + */ + UNKNOWN(-1), + + /** + * Success, no error. + */ + SUCCESS(0), + + /** + * 
+     * All Kinds of Timeout(Including Election_timeout, Timeout_now, Stepdown_timeout)
+     *
+ *

+ * ERAFTTIMEDOUT = 10001; + */ + ERAFTTIMEDOUT(10001), + + /** + * 

+     * Bad User State Machine
+     *
+ *

+ * ESTATEMACHINE = 10002; + */ + ESTATEMACHINE(10002), + + /** + * 

+     * Catchup Failed
+     *
+ *

+ * ECATCHUP = 10003; + */ + ECATCHUP(10003), + + /** + * 

+     * Trigger step_down(Not All)
+     *
+ *

+ * ELEADERREMOVED = 10004; + */ + ELEADERREMOVED(10004), + + /** + * 

+     * Leader Is Not In The New Configuration
+     *
+ *

+ * ESETPEER = 10005; + */ + ESETPEER(10005), + + /** + * 

+     * Shut_down
+     *
+ *

+ * ENODESHUTDOWN = 10006; + */ + ENODESHUTDOWN(10006), + + /** + * 

+     * Receive Higher Term Requests
+     *
+ *

+ * EHIGHERTERMREQUEST = 10007; + */ + EHIGHERTERMREQUEST(10007), + + /** + * 

+     * Receive Higher Term Response
+     *
+ *

+ * EHIGHERTERMRESPONSE = 10008; + */ + EHIGHERTERMRESPONSE(10008), + + /** + * 

+     * Node Is In Error
+     *
+ *

+ * EBADNODE = 10009; + */ + EBADNODE(10009), + + /** + * 

+     * Node Votes For Some Candidate
+     *
+ *

+ * EVOTEFORCANDIDATE = 10010; + */ + EVOTEFORCANDIDATE(10010), + + /** + * 

+     * Follower(without leader) or Candidate Receives
+     *
+ *

+ * ENEWLEADER = 10011; + */ + ENEWLEADER(10011), + + /** + * 

+     * Append_entries/Install_snapshot Request from a new leader
+     *
+ *

+ * ELEADERCONFLICT = 10012; + */ + ELEADERCONFLICT(10012), + + /** + * 

+     * Trigger on_leader_stop
+     *
+ *

+ * ETRANSFERLEADERSHIP = 10013; + */ + ETRANSFERLEADERSHIP(10013), + + /** + * 

+     * The log at the given index is deleted
+     *
+ *

+ * ELOGDELETED = 10014; + */ + ELOGDELETED(10014), + + /** + * 

+     * No available user log to read
+     *
+ *

+ * ENOMOREUSERLOG = 10015; + */ + ENOMOREUSERLOG(10015), + + /* other non-raft error codes 1000~10000 */ + /** + * Invalid rpc request + */ + EREQUEST(1000), + + /** + * Task is stopped + */ + ESTOP(1001), + + /** + * Retry again + */ + EAGAIN(1002), + + /** + * Interrupted + */ + EINTR(1003), + + /** + * Internal exception + */ + EINTERNAL(1004), + + /** + * Task is canceled + */ + ECANCELED(1005), + + /** + * Host is down + */ + EHOSTDOWN(1006), + + /** + * Service is shutdown + */ + ESHUTDOWN(1007), + + /** + * Permission issue + */ + EPERM(1008), + + /** + * Server is in busy state + */ + EBUSY(1009), + + /** + * Timed out + */ + ETIMEDOUT(1010), + + /** + * Data is stale + */ + ESTALE(1011), + + /** + * Something not found + */ + ENOENT(1012), + + /** + * File/folder already exists + */ + EEXISTS(1013), + + /** + * IO error + */ + EIO(1014), + + /** + * Invalid value. + */ + EINVAL(1015), + + /** + * Permission denied + */ + EACCES(1016); + + private static final Map RAFT_ERROR_MAP = new HashMap<>(); + + static { + for (final RaftError error : RaftError.values()) { + RAFT_ERROR_MAP.put(error.getNumber(), error); + } + } + + public final int getNumber() { + return this.value; + } + + public static RaftError forNumber(final int value) { + return RAFT_ERROR_MAP.getOrDefault(value, UNKNOWN); + } + + public static String describeCode(final int code) { + RaftError e = forNumber(code); + return e != null ? e.name() : ""; + } + + private final int value; + + RaftError(final int value) { + this.value = value; + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftException.java new file mode 100644 index 0000000..87499cd --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RaftException.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.EnumOutter; + +/** + * A raft exception. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-14 10:41:21 AM + */ +public class RaftException extends Throwable { + + private static final long serialVersionUID = -1533343555230409704L; + + /** + * Error type + */ + private EnumOutter.ErrorType type; + /** Error status*/ + private Status status = Status.OK(); + + public RaftException(EnumOutter.ErrorType type) { + super(type.name()); + this.type = type; + } + + public RaftException(EnumOutter.ErrorType type, Status status) { + super(status != null ? status.getErrorMsg() : type.name()); + this.type = type; + this.status = status; + } + + public RaftException(EnumOutter.ErrorType type, RaftError err, String fmt, Object... args) { + super(String.format(fmt, args)); + this.type = type; + this.status = new Status(err, fmt, args); + } + + public RaftException() { + this.type = EnumOutter.ErrorType.ERROR_TYPE_NONE; + this.status = Status.OK(); + } + + public EnumOutter.ErrorType getType() { + return this.type; + } + + public void setType(EnumOutter.ErrorType type) { + this.type = type; + } + + public Status getStatus() { + return this.status; + } + + public void setStatus(Status status) { + this.status = status; + } + + @Override + public String toString() { + return "Error [type=" + this.type + ", status=" + this.status + "]"; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RemotingException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RemotingException.java new file mode 100644 index 0000000..29381ac --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RemotingException.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Exception for default remoting problems. + * + * @author jiachun.fjc + */ +public class RemotingException extends Exception { + + private static final long serialVersionUID = -6326244159775972292L; + + public RemotingException() { + } + + public RemotingException(String message) { + super(message); + } + + public RemotingException(String message, Throwable cause) { + super(message, cause); + } + + public RemotingException(Throwable cause) { + super(cause); + } + + public RemotingException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RetryAgainException.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RetryAgainException.java new file mode 100644 index 0000000..e3f325f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/error/RetryAgainException.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.error; + +/** + * Retry request exception without stack tracing. + * + * @author dennis + */ +public class RetryAgainException extends Exception { + + private static final long serialVersionUID = 8690835003361525337L; + + public RetryAgainException() { + super(); + } + + public RetryAgainException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } + + public RetryAgainException(String message, Throwable cause) { + super(message, cause); + } + + public RetryAgainException(String message) { + super(message); + } + + public RetryAgainException(Throwable cause) { + super(cause); + } + + @Override + public Throwable fillInStackTrace() { + return this; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ApplyTaskMode.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ApplyTaskMode.java new file mode 100644 index 0000000..8c62067 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ApplyTaskMode.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +/** + * Apply task in blocking or non-blocking mode. + * @author boyan(boyan@antfin.com) + * + */ +public enum ApplyTaskMode { + Blocking, NonBlocking +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BallotBoxOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BallotBoxOptions.java new file mode 100644 index 0000000..ae92cf5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BallotBoxOptions.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.closure.ClosureQueue; + +/** + * Ballot box options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:58:36 PM + */ +public class BallotBoxOptions { + + private FSMCaller waiter; + private ClosureQueue closureQueue; + + public FSMCaller getWaiter() { + return this.waiter; + } + + public void setWaiter(FSMCaller waiter) { + this.waiter = waiter; + } + + public ClosureQueue getClosureQueue() { + return this.closureQueue; + } + + public void setClosureQueue(ClosureQueue closureQueue) { + this.closureQueue = closureQueue; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BootstrapOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BootstrapOptions.java new file mode 100644 index 0000000..c726da1 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/BootstrapOptions.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.JRaftServiceFactory; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.util.JRaftServiceLoader; + +/** + * Bootstrap options + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:58:45 PM + */ +public class BootstrapOptions { + + public static final JRaftServiceFactory defaultServiceFactory = JRaftServiceLoader.load(JRaftServiceFactory.class) // + .first(); + + // Containing the initial member of this raft group + // Default: empty conf + private Configuration groupConf; + + // The index of the last index which the dumping snapshot contains + // Default: 0 + private long lastLogIndex = 0L; + + // The specific StateMachine which is going to dump the first snapshot + // If last_log_index isn't 0, fsm must be a valid instance. + // Default: NULL + private StateMachine fsm; + + // Describe a specific LogStorage in format ${type}://${parameters} + private String logUri; + + // Describe a specific RaftMetaStorage in format ${type}://${parameters} + private String raftMetaUri; + + // Describe a specific SnapshotStorage in format ${type}://${parameters} + private String snapshotUri; + + // Whether to enable metrics for node. + private boolean enableMetrics = false; + + /** + * Custom service factory. + */ + private JRaftServiceFactory serviceFactory = defaultServiceFactory; + + public JRaftServiceFactory getServiceFactory() { + return serviceFactory; + } + + public void setServiceFactory(JRaftServiceFactory serviceFactory) { + this.serviceFactory = serviceFactory; + } + + public void setEnableMetrics(boolean enableMetrics) { + this.enableMetrics = enableMetrics; + } + + public boolean isEnableMetrics() { + return enableMetrics; + } + + public Configuration getGroupConf() { + return this.groupConf; + } + + public void setGroupConf(Configuration groupConf) { + this.groupConf = groupConf; + } + + public long getLastLogIndex() { + return this.lastLogIndex; + } + + public void setLastLogIndex(long lastLogIndex) { + this.lastLogIndex = lastLogIndex; + } + + public StateMachine getFsm() { + return this.fsm; + } + + public void setFsm(StateMachine fsm) { + this.fsm = fsm; + } + + public String getLogUri() { + return this.logUri; + } + + public void setLogUri(String logUri) { + this.logUri = logUri; + } + + public String getRaftMetaUri() { + return this.raftMetaUri; + } + + public void setRaftMetaUri(String raftMetaUri) { + this.raftMetaUri = raftMetaUri; + } + + public String getSnapshotUri() { + return this.snapshotUri; + } + + public void setSnapshotUri(String snapshotUri) { + this.snapshotUri = snapshotUri; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CliOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CliOptions.java new file mode 100644 index 0000000..6719722 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CliOptions.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +/** + * Cli service options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 3:25:59 PM + */ +public class CliOptions extends RpcOptions { + + private int timeoutMs; + private int maxRetry; + + public int getTimeoutMs() { + return this.timeoutMs; + } + + public void setTimeoutMs(int timeoutMs) { + this.timeoutMs = timeoutMs; + } + + public int getMaxRetry() { + return this.maxRetry; + } + + public void setMaxRetry(int maxRetry) { + this.maxRetry = maxRetry; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CopyOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CopyOptions.java new file mode 100644 index 0000000..9151fd7 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/CopyOptions.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +/** + * Copier options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:58:53 PM + */ +public class CopyOptions { + + private int maxRetry = 3; + private long retryIntervalMs = 1000L; + private int timeoutMs = 10 * 1000; + + public int getMaxRetry() { + return this.maxRetry; + } + + public void setMaxRetry(int maxRetry) { + this.maxRetry = maxRetry; + } + + public long getRetryIntervalMs() { + return this.retryIntervalMs; + } + + public void setRetryIntervalMs(long retryIntervalMs) { + this.retryIntervalMs = retryIntervalMs; + } + + public int getTimeoutMs() { + return this.timeoutMs; + } + + public void setTimeoutMs(int timeoutMs) { + this.timeoutMs = timeoutMs; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/FSMCallerOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/FSMCallerOptions.java new file mode 100644 index 0000000..f4f1a9a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/FSMCallerOptions.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.closure.ClosureQueue; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.storage.LogManager; + +/** + * FSM caller options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:59:02 PM + */ +public class FSMCallerOptions { + private LogManager logManager; + private StateMachine fsm; + private Closure afterShutdown; + private LogId bootstrapId; + private ClosureQueue closureQueue; + private NodeImpl node; + /** + * disruptor buffer size. + */ + private int disruptorBufferSize = 1024; + + public int getDisruptorBufferSize() { + return this.disruptorBufferSize; + } + + public void setDisruptorBufferSize(int disruptorBufferSize) { + this.disruptorBufferSize = disruptorBufferSize; + } + + public NodeImpl getNode() { + return this.node; + } + + public void setNode(NodeImpl node) { + this.node = node; + } + + public ClosureQueue getClosureQueue() { + return this.closureQueue; + } + + public void setClosureQueue(ClosureQueue closureQueue) { + this.closureQueue = closureQueue; + } + + public LogManager getLogManager() { + return this.logManager; + } + + public void setLogManager(LogManager logManager) { + this.logManager = logManager; + } + + public StateMachine getFsm() { + return this.fsm; + } + + public void setFsm(StateMachine fsm) { + this.fsm = fsm; + } + + public Closure getAfterShutdown() { + return this.afterShutdown; + } + + public void setAfterShutdown(Closure afterShutdown) { + this.afterShutdown = afterShutdown; + } + + public LogId getBootstrapId() { + return this.bootstrapId; + } + + public void setBootstrapId(LogId bootstrapId) { + this.bootstrapId = bootstrapId; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogManagerOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogManagerOptions.java new file mode 100644 index 0000000..9219076 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogManagerOptions.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.conf.ConfigurationManager; +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; +import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory; +import com.alipay.sofa.jraft.storage.LogStorage; + +/** + * Options for log manager. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-13 5:15:15 PM + */ +public class LogManagerOptions { + + private LogStorage logStorage; + private ConfigurationManager configurationManager; + private FSMCaller fsmCaller; + private int disruptorBufferSize = 1024; + private RaftOptions raftOptions; + private NodeMetrics nodeMetrics; + private LogEntryCodecFactory logEntryCodecFactory = LogEntryV2CodecFactory.getInstance(); + + public LogEntryCodecFactory getLogEntryCodecFactory() { + return this.logEntryCodecFactory; + } + + public void setLogEntryCodecFactory(final LogEntryCodecFactory logEntryCodecFactory) { + this.logEntryCodecFactory = logEntryCodecFactory; + } + + public NodeMetrics getNodeMetrics() { + return this.nodeMetrics; + } + + public void setNodeMetrics(final NodeMetrics nodeMetrics) { + this.nodeMetrics = nodeMetrics; + } + + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + public void setRaftOptions(final RaftOptions raftOptions) { + this.raftOptions = raftOptions; + } + + public int getDisruptorBufferSize() { + return this.disruptorBufferSize; + } + + public void setDisruptorBufferSize(final int disruptorBufferSize) { + this.disruptorBufferSize = disruptorBufferSize; + } + + public LogStorage getLogStorage() { + return this.logStorage; + } + + public void setLogStorage(final LogStorage logStorage) { + this.logStorage = logStorage; + } + + public ConfigurationManager getConfigurationManager() { + return this.configurationManager; + } + + public void setConfigurationManager(final ConfigurationManager configurationManager) { + this.configurationManager = configurationManager; + } + + public FSMCaller getFsmCaller() { + return this.fsmCaller; + } + + public void setFsmCaller(final FSMCaller fsmCaller) { + this.fsmCaller = fsmCaller; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogStorageOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogStorageOptions.java new file mode 100644 index 0000000..b3620b2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/LogStorageOptions.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.conf.ConfigurationManager; +import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory; + +/** + * Log storage initialize options + * @author boyan(boyan@antfin.com) + * + */ +public class LogStorageOptions { + + private ConfigurationManager configurationManager; + + private LogEntryCodecFactory logEntryCodecFactory; + + public ConfigurationManager getConfigurationManager() { + return this.configurationManager; + } + + public void setConfigurationManager(final ConfigurationManager configurationManager) { + this.configurationManager = configurationManager; + } + + public LogEntryCodecFactory getLogEntryCodecFactory() { + return this.logEntryCodecFactory; + } + + public void setLogEntryCodecFactory(final LogEntryCodecFactory logEntryCodecFactory) { + this.logEntryCodecFactory = logEntryCodecFactory; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/NodeOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/NodeOptions.java new file mode 100644 index 0000000..442d699 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/NodeOptions.java @@ -0,0 +1,458 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.remoting.util.StringUtils; +import com.alipay.sofa.jraft.JRaftServiceFactory; +import com.alipay.sofa.jraft.StateMachine; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.core.ElectionPriority; +import com.alipay.sofa.jraft.storage.SnapshotThrottle; +import com.alipay.sofa.jraft.util.Copiable; +import com.alipay.sofa.jraft.util.JRaftServiceLoader; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Node options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:59:12 PM + */ +public class NodeOptions extends RpcOptions implements Copiable { + + public static final JRaftServiceFactory defaultServiceFactory = JRaftServiceLoader.load(JRaftServiceFactory.class) // + .first(); + + // A follower would become a candidate if it doesn't receive any message + // from the leader in |election_timeout_ms| milliseconds + // Default: 1000 (1s) + private int electionTimeoutMs = 1000; // follower to candidate timeout + + // One node's local priority value would be set to | electionPriority | + // value when it starts up.If this value is set to 0,the node will never be a leader. + // If this node doesn't support priority election,then set this value to -1. + // Default: -1 + private int electionPriority = ElectionPriority.Disabled; + + // If next leader is not elected until next election timeout, it exponentially + // decay its local target priority, for example target_priority = target_priority - gap + // Default: 10 + private int decayPriorityGap = 10; + + // Leader lease time's ratio of electionTimeoutMs, + // To minimize the effects of clock drift, we should make that: + // clockDrift + leaderLeaseTimeoutMs < electionTimeout + // Default: 90, Max: 100 + private int leaderLeaseTimeRatio = 90; + + // A snapshot saving would be triggered every |snapshot_interval_s| seconds + // if this was reset as a positive number + // If |snapshot_interval_s| <= 0, the time based snapshot would be disabled. + // + // Default: 3600 (1 hour) + private int snapshotIntervalSecs = 3600; + + // A snapshot saving would be triggered every |snapshot_interval_s| seconds, + // and at this moment when state machine's lastAppliedIndex value + // minus lastSnapshotId value is greater than snapshotLogIndexMargin value, + // the snapshot action will be done really. + // If |snapshotLogIndexMargin| <= 0, the distance based snapshot would be disable. + // + // Default: 0 + private int snapshotLogIndexMargin = 0; + + // We will regard a adding peer as caught up if the margin between the + // last_log_index of this peer and the last_log_index of leader is less than + // |catchup_margin| + // + // Default: 1000 + private int catchupMargin = 1000; + + // If node is starting from a empty environment (both LogStorage and + // SnapshotStorage are empty), it would use |initial_conf| as the + // configuration of the group, otherwise it would load configuration from + // the existing environment. + // + // Default: A empty group + private Configuration initialConf = new Configuration(); + + // The specific StateMachine implemented your business logic, which must be + // a valid instance. + private StateMachine fsm; + + // Describe a specific LogStorage in format ${type}://${parameters} + private String logUri; + + // Describe a specific RaftMetaStorage in format ${type}://${parameters} + private String raftMetaUri; + + // Describe a specific SnapshotStorage in format ${type}://${parameters} + private String snapshotUri; + + // If enable, we will filter duplicate files before copy remote snapshot, + // to avoid useless transmission. Two files in local and remote are duplicate, + // only if they has the same filename and the same checksum (stored in file meta). + // Default: false + private boolean filterBeforeCopyRemote = false; + + // If non-null, we will pass this throughput_snapshot_throttle to SnapshotExecutor + // Default: NULL + // scoped_refptr* snapshot_throttle; + + // If true, RPCs through raft_cli will be denied. + // Default: false + private boolean disableCli = false; + + /** + * Whether use global timer pool, if true, the {@code timerPoolSize} will be invalid. + */ + private boolean sharedTimerPool = false; + /** + * Timer manager thread pool size + */ + private int timerPoolSize = Utils.cpus() * 3 > 20 ? 20 : Utils.cpus() * 3; + + /** + * CLI service request RPC executor pool size, use default executor if -1. + */ + private int cliRpcThreadPoolSize = Utils.cpus(); + /** + * RAFT request RPC executor pool size, use default executor if -1. + */ + private int raftRpcThreadPoolSize = Utils.cpus() * 6; + /** + * Whether to enable metrics for node. + */ + private boolean enableMetrics = false; + + /** + * If non-null, we will pass this SnapshotThrottle to SnapshotExecutor + * Default: NULL + */ + private SnapshotThrottle snapshotThrottle; + + /** + * Whether use global election timer + */ + private boolean sharedElectionTimer = false; + /** + * Whether use global vote timer + */ + private boolean sharedVoteTimer = false; + /** + * Whether use global step down timer + */ + private boolean sharedStepDownTimer = false; + /** + * Whether use global snapshot timer + */ + private boolean sharedSnapshotTimer = false; + + /** + * Custom service factory. + */ + private JRaftServiceFactory serviceFactory = defaultServiceFactory; + + /** + * Apply task in blocking or non-blocking mode, ApplyTaskMode.NonBlocking by default. + */ + private ApplyTaskMode applyTaskMode = ApplyTaskMode.NonBlocking; + + public ApplyTaskMode getApplyTaskMode() { + return this.applyTaskMode; + } + + public void setApplyTaskMode(final ApplyTaskMode applyTaskMode) { + this.applyTaskMode = applyTaskMode; + } + + public JRaftServiceFactory getServiceFactory() { + return this.serviceFactory; + } + + public void setServiceFactory(final JRaftServiceFactory serviceFactory) { + this.serviceFactory = serviceFactory; + } + + public SnapshotThrottle getSnapshotThrottle() { + return this.snapshotThrottle; + } + + public void setSnapshotThrottle(final SnapshotThrottle snapshotThrottle) { + this.snapshotThrottle = snapshotThrottle; + } + + public void setEnableMetrics(final boolean enableMetrics) { + this.enableMetrics = enableMetrics; + } + + /** + * Raft options + */ + private RaftOptions raftOptions = new RaftOptions(); + + public int getCliRpcThreadPoolSize() { + return this.cliRpcThreadPoolSize; + } + + public void setCliRpcThreadPoolSize(final int cliRpcThreadPoolSize) { + this.cliRpcThreadPoolSize = cliRpcThreadPoolSize; + } + + public boolean isEnableMetrics() { + return this.enableMetrics; + } + + public int getRaftRpcThreadPoolSize() { + return this.raftRpcThreadPoolSize; + } + + public void setRaftRpcThreadPoolSize(final int raftRpcThreadPoolSize) { + this.raftRpcThreadPoolSize = raftRpcThreadPoolSize; + } + + public boolean isSharedTimerPool() { + return this.sharedTimerPool; + } + + public void setSharedTimerPool(final boolean sharedTimerPool) { + this.sharedTimerPool = sharedTimerPool; + } + + public int getTimerPoolSize() { + return this.timerPoolSize; + } + + public void setTimerPoolSize(final int timerPoolSize) { + this.timerPoolSize = timerPoolSize; + } + + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + public void setRaftOptions(final RaftOptions raftOptions) { + this.raftOptions = raftOptions; + } + + public void validate() { + if (StringUtils.isBlank(this.logUri)) { + throw new IllegalArgumentException("Blank logUri"); + } + if (StringUtils.isBlank(this.raftMetaUri)) { + throw new IllegalArgumentException("Blank raftMetaUri"); + } + if (this.fsm == null) { + throw new IllegalArgumentException("Null stateMachine"); + } + } + + public int getElectionPriority() { + return this.electionPriority; + } + + public void setElectionPriority(final int electionPriority) { + this.electionPriority = electionPriority; + } + + public int getDecayPriorityGap() { + return this.decayPriorityGap; + } + + public void setDecayPriorityGap(final int decayPriorityGap) { + this.decayPriorityGap = decayPriorityGap; + } + + public int getElectionTimeoutMs() { + return this.electionTimeoutMs; + } + + public void setElectionTimeoutMs(final int electionTimeoutMs) { + this.electionTimeoutMs = electionTimeoutMs; + } + + public int getLeaderLeaseTimeRatio() { + return this.leaderLeaseTimeRatio; + } + + public void setLeaderLeaseTimeRatio(final int leaderLeaseTimeRatio) { + if (leaderLeaseTimeRatio <= 0 || leaderLeaseTimeRatio > 100) { + throw new IllegalArgumentException("leaderLeaseTimeRatio: " + leaderLeaseTimeRatio + + " (expected: 0 < leaderLeaseTimeRatio <= 100)"); + } + this.leaderLeaseTimeRatio = leaderLeaseTimeRatio; + } + + public int getLeaderLeaseTimeoutMs() { + return this.electionTimeoutMs * this.leaderLeaseTimeRatio / 100; + } + + public int getSnapshotIntervalSecs() { + return this.snapshotIntervalSecs; + } + + public void setSnapshotIntervalSecs(final int snapshotIntervalSecs) { + this.snapshotIntervalSecs = snapshotIntervalSecs; + } + + public int getSnapshotLogIndexMargin() { + return this.snapshotLogIndexMargin; + } + + public void setSnapshotLogIndexMargin(final int snapshotLogIndexMargin) { + this.snapshotLogIndexMargin = snapshotLogIndexMargin; + } + + public int getCatchupMargin() { + return this.catchupMargin; + } + + public void setCatchupMargin(final int catchupMargin) { + this.catchupMargin = catchupMargin; + } + + public Configuration getInitialConf() { + return this.initialConf; + } + + public void setInitialConf(final Configuration initialConf) { + this.initialConf = initialConf; + } + + public StateMachine getFsm() { + return this.fsm; + } + + public void setFsm(final StateMachine fsm) { + this.fsm = fsm; + } + + public String getLogUri() { + return this.logUri; + } + + public void setLogUri(final String logUri) { + this.logUri = logUri; + } + + public String getRaftMetaUri() { + return this.raftMetaUri; + } + + public void setRaftMetaUri(final String raftMetaUri) { + this.raftMetaUri = raftMetaUri; + } + + public String getSnapshotUri() { + return this.snapshotUri; + } + + public void setSnapshotUri(final String snapshotUri) { + this.snapshotUri = snapshotUri; + } + + public boolean isFilterBeforeCopyRemote() { + return this.filterBeforeCopyRemote; + } + + public void setFilterBeforeCopyRemote(final boolean filterBeforeCopyRemote) { + this.filterBeforeCopyRemote = filterBeforeCopyRemote; + } + + public boolean isDisableCli() { + return this.disableCli; + } + + public void setDisableCli(final boolean disableCli) { + this.disableCli = disableCli; + } + + public boolean isSharedElectionTimer() { + return this.sharedElectionTimer; + } + + public void setSharedElectionTimer(final boolean sharedElectionTimer) { + this.sharedElectionTimer = sharedElectionTimer; + } + + public boolean isSharedVoteTimer() { + return this.sharedVoteTimer; + } + + public void setSharedVoteTimer(final boolean sharedVoteTimer) { + this.sharedVoteTimer = sharedVoteTimer; + } + + public boolean isSharedStepDownTimer() { + return this.sharedStepDownTimer; + } + + public void setSharedStepDownTimer(final boolean sharedStepDownTimer) { + this.sharedStepDownTimer = sharedStepDownTimer; + } + + public boolean isSharedSnapshotTimer() { + return this.sharedSnapshotTimer; + } + + public void setSharedSnapshotTimer(final boolean sharedSnapshotTimer) { + this.sharedSnapshotTimer = sharedSnapshotTimer; + } + + @Override + public NodeOptions copy() { + final NodeOptions nodeOptions = new NodeOptions(); + nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs); + nodeOptions.setElectionPriority(this.electionPriority); + nodeOptions.setDecayPriorityGap(this.decayPriorityGap); + nodeOptions.setSnapshotIntervalSecs(this.snapshotIntervalSecs); + nodeOptions.setSnapshotLogIndexMargin(this.snapshotLogIndexMargin); + nodeOptions.setCatchupMargin(this.catchupMargin); + nodeOptions.setFilterBeforeCopyRemote(this.filterBeforeCopyRemote); + nodeOptions.setDisableCli(this.disableCli); + nodeOptions.setSharedTimerPool(this.sharedTimerPool); + nodeOptions.setTimerPoolSize(this.timerPoolSize); + nodeOptions.setCliRpcThreadPoolSize(this.cliRpcThreadPoolSize); + nodeOptions.setRaftRpcThreadPoolSize(this.raftRpcThreadPoolSize); + nodeOptions.setEnableMetrics(this.enableMetrics); + nodeOptions.setRaftOptions(this.raftOptions == null ? new RaftOptions() : this.raftOptions.copy()); + nodeOptions.setSharedElectionTimer(this.sharedElectionTimer); + nodeOptions.setSharedVoteTimer(this.sharedVoteTimer); + nodeOptions.setSharedStepDownTimer(this.sharedStepDownTimer); + nodeOptions.setSharedSnapshotTimer(this.sharedSnapshotTimer); + return nodeOptions; + } + + @Override + public String toString() { + return "NodeOptions{" + "electionTimeoutMs=" + this.electionTimeoutMs + ", electionPriority=" + + this.electionPriority + ", decayPriorityGap=" + this.decayPriorityGap + ", leaderLeaseTimeRatio=" + + this.leaderLeaseTimeRatio + ", snapshotIntervalSecs=" + this.snapshotIntervalSecs + + ", snapshotLogIndexMargin=" + this.snapshotLogIndexMargin + ", catchupMargin=" + this.catchupMargin + + ", initialConf=" + this.initialConf + ", fsm=" + this.fsm + ", logUri='" + this.logUri + '\'' + + ", raftMetaUri='" + this.raftMetaUri + '\'' + ", snapshotUri='" + this.snapshotUri + '\'' + + ", filterBeforeCopyRemote=" + this.filterBeforeCopyRemote + ", disableCli=" + this.disableCli + + ", sharedTimerPool=" + this.sharedTimerPool + ", timerPoolSize=" + this.timerPoolSize + + ", cliRpcThreadPoolSize=" + this.cliRpcThreadPoolSize + ", raftRpcThreadPoolSize=" + + this.raftRpcThreadPoolSize + ", enableMetrics=" + this.enableMetrics + ", snapshotThrottle=" + + this.snapshotThrottle + ", sharedElectionTimer=" + this.sharedElectionTimer + ", sharedVoteTimer=" + + this.sharedVoteTimer + ", sharedStepDownTimer=" + this.sharedStepDownTimer + ", sharedSnapshotTimer=" + + this.sharedSnapshotTimer + ", serviceFactory=" + this.serviceFactory + ", applyTaskMode=" + + this.applyTaskMode + ", raftOptions=" + this.raftOptions + "} " + super.toString(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftMetaStorageOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftMetaStorageOptions.java new file mode 100644 index 0000000..c5984f2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftMetaStorageOptions.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.core.NodeImpl; + +/** + * Raft meta storage options + * @author dennis + * + */ +public class RaftMetaStorageOptions { + private NodeImpl node; + + public NodeImpl getNode() { + return this.node; + } + + public void setNode(NodeImpl node) { + this.node = node; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftOptions.java new file mode 100644 index 0000000..82dfa8d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RaftOptions.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.util.Copiable; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; + +/** + * Raft options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:38:40 PM + */ +public class RaftOptions implements Copiable { + + /** Maximum of block size per RPC */ + private int maxByteCountPerRpc = 128 * 1024; + /** File service check hole switch, default disable */ + private boolean fileCheckHole = false; + /** The maximum number of entries in AppendEntriesRequest */ + private int maxEntriesSize = 1024; + /** The maximum byte size of AppendEntriesRequest */ + private int maxBodySize = 512 * 1024; + /** Flush buffer to LogStorage if the buffer size reaches the limit */ + private int maxAppendBufferSize = 256 * 1024; + /** Maximum election delay time allowed by user */ + private int maxElectionDelayMs = 1000; + /** Raft election:heartbeat timeout factor */ + private int electionHeartbeatFactor = 10; + /** Maximum number of tasks that can be applied in a batch */ + private int applyBatch = 32; + /** Call fsync when need */ + private boolean sync = true; + /** Sync log meta, snapshot meta and raft meta */ + private boolean syncMeta = false; + /** Statistics to analyze the performance of db */ + private boolean openStatistics = true; + /** Whether to enable replicator pipeline. */ + private boolean replicatorPipeline = true; + /** The maximum replicator pipeline in-flight requests/responses, only valid when enable replicator pipeline. */ + private int maxReplicatorInflightMsgs = 256; + /** Internal disruptor buffers size for Node/FSMCaller/LogManager etc. */ + private int disruptorBufferSize = 16384; + /** + * The maximum timeout in seconds to wait when publishing events into disruptor, default is 10 seconds. + * If the timeout happens, it may halt the node. + * */ + private int disruptorPublishEventWaitTimeoutSecs = 10; + /** + * When true, validate log entry checksum when transferring the log entry from disk or network, default is false. + * If true, it would hurt the performance of JRAft but gain the data safety. + * @since 1.2.6 + */ + private boolean enableLogEntryChecksum = false; + + /** + * ReadOnlyOption specifies how the read only request is processed. + * + * {@link ReadOnlyOption#ReadOnlySafe} guarantees the linearizability of the read only request by + * communicating with the quorum. It is the default and suggested option. + + * {@link ReadOnlyOption#ReadOnlyLeaseBased} ensures linearizability of the read only request by + * relying on the leader lease. It can be affected by clock drift. + * If the clock drift is unbounded, leader might keep the lease longer than it + * should (clock can move backward/pause without any bound). ReadIndex is not safe + * in that case. + */ + private ReadOnlyOption readOnlyOptions = ReadOnlyOption.ReadOnlySafe; + + /** + * Read index read need compare current node's apply index with leader's commit index. + * Only current node's apply index catch up leader's commit index, then call back success to read index closure. + * Therefore, there is a waiting time. The default wait timeout is 2s. It means that the waiting time + * over 2s, then call back failure to read index closure. If current node occur problem, it's apply index maybe + * behind leader's commit index. In read index timeout, it can't catch up, the timeout is waste. + * Here supply a config to fix it. If the gap greater than maxReadIndexLag, fail fast to call back failure + * read index closure. + * @since 1.4.0 + */ + private int maxReadIndexLag = -1; + + /** + * Candidate steps down when election reaching timeout, default is true(enabled). + * @since 1.3.0 + */ + private boolean stepDownWhenVoteTimedout = true; + + public boolean isStepDownWhenVoteTimedout() { + return this.stepDownWhenVoteTimedout; + } + + public void setStepDownWhenVoteTimedout(final boolean stepDownWhenVoteTimeout) { + this.stepDownWhenVoteTimedout = stepDownWhenVoteTimeout; + } + + public int getDisruptorPublishEventWaitTimeoutSecs() { + return this.disruptorPublishEventWaitTimeoutSecs; + } + + public void setDisruptorPublishEventWaitTimeoutSecs(final int disruptorPublishEventWaitTimeoutSecs) { + this.disruptorPublishEventWaitTimeoutSecs = disruptorPublishEventWaitTimeoutSecs; + } + + public boolean isEnableLogEntryChecksum() { + return this.enableLogEntryChecksum; + } + + public void setEnableLogEntryChecksum(final boolean enableLogEntryChecksumValidation) { + this.enableLogEntryChecksum = enableLogEntryChecksumValidation; + } + + public ReadOnlyOption getReadOnlyOptions() { + return this.readOnlyOptions; + } + + public void setReadOnlyOptions(final ReadOnlyOption readOnlyOptions) { + this.readOnlyOptions = readOnlyOptions; + } + + public int getMaxReadIndexLag() { + return maxReadIndexLag; + } + + public void setMaxReadIndexLag(int maxReadIndexLag) { + this.maxReadIndexLag = maxReadIndexLag; + } + + public boolean isReplicatorPipeline() { + return this.replicatorPipeline && RpcFactoryHelper.rpcFactory().isReplicatorPipelineEnabled(); + } + + public void setReplicatorPipeline(final boolean replicatorPipeline) { + this.replicatorPipeline = replicatorPipeline; + } + + public int getMaxReplicatorInflightMsgs() { + return this.maxReplicatorInflightMsgs; + } + + public void setMaxReplicatorInflightMsgs(final int maxReplicatorPiplelinePendingResponses) { + this.maxReplicatorInflightMsgs = maxReplicatorPiplelinePendingResponses; + } + + public int getDisruptorBufferSize() { + return this.disruptorBufferSize; + } + + public void setDisruptorBufferSize(final int disruptorBufferSize) { + this.disruptorBufferSize = disruptorBufferSize; + } + + public int getMaxByteCountPerRpc() { + return this.maxByteCountPerRpc; + } + + public void setMaxByteCountPerRpc(final int maxByteCountPerRpc) { + this.maxByteCountPerRpc = maxByteCountPerRpc; + } + + public boolean isFileCheckHole() { + return this.fileCheckHole; + } + + public void setFileCheckHole(final boolean fileCheckHole) { + this.fileCheckHole = fileCheckHole; + } + + public int getMaxEntriesSize() { + return this.maxEntriesSize; + } + + public void setMaxEntriesSize(final int maxEntriesSize) { + this.maxEntriesSize = maxEntriesSize; + } + + public int getMaxBodySize() { + return this.maxBodySize; + } + + public void setMaxBodySize(final int maxBodySize) { + this.maxBodySize = maxBodySize; + } + + public int getMaxAppendBufferSize() { + return this.maxAppendBufferSize; + } + + public void setMaxAppendBufferSize(final int maxAppendBufferSize) { + this.maxAppendBufferSize = maxAppendBufferSize; + } + + public int getMaxElectionDelayMs() { + return this.maxElectionDelayMs; + } + + public void setMaxElectionDelayMs(final int maxElectionDelayMs) { + this.maxElectionDelayMs = maxElectionDelayMs; + } + + public int getElectionHeartbeatFactor() { + return this.electionHeartbeatFactor; + } + + public void setElectionHeartbeatFactor(final int electionHeartbeatFactor) { + this.electionHeartbeatFactor = electionHeartbeatFactor; + } + + public int getApplyBatch() { + return this.applyBatch; + } + + public void setApplyBatch(final int applyBatch) { + this.applyBatch = applyBatch; + } + + public boolean isSync() { + return this.sync; + } + + public void setSync(final boolean sync) { + this.sync = sync; + } + + public boolean isSyncMeta() { + return this.sync || this.syncMeta; + } + + public void setSyncMeta(final boolean syncMeta) { + this.syncMeta = syncMeta; + } + + public boolean isOpenStatistics() { + return this.openStatistics; + } + + public void setOpenStatistics(final boolean openStatistics) { + this.openStatistics = openStatistics; + } + + @Override + public RaftOptions copy() { + final RaftOptions raftOptions = new RaftOptions(); + raftOptions.setMaxByteCountPerRpc(this.maxByteCountPerRpc); + raftOptions.setFileCheckHole(this.fileCheckHole); + raftOptions.setMaxEntriesSize(this.maxEntriesSize); + raftOptions.setMaxBodySize(this.maxBodySize); + raftOptions.setMaxAppendBufferSize(this.maxAppendBufferSize); + raftOptions.setMaxElectionDelayMs(this.maxElectionDelayMs); + raftOptions.setElectionHeartbeatFactor(this.electionHeartbeatFactor); + raftOptions.setApplyBatch(this.applyBatch); + raftOptions.setSync(this.sync); + raftOptions.setSyncMeta(this.syncMeta); + raftOptions.setOpenStatistics(this.openStatistics); + raftOptions.setReplicatorPipeline(this.replicatorPipeline); + raftOptions.setMaxReplicatorInflightMsgs(this.maxReplicatorInflightMsgs); + raftOptions.setDisruptorBufferSize(this.disruptorBufferSize); + raftOptions.setDisruptorPublishEventWaitTimeoutSecs(this.disruptorPublishEventWaitTimeoutSecs); + raftOptions.setEnableLogEntryChecksum(this.enableLogEntryChecksum); + raftOptions.setReadOnlyOptions(this.readOnlyOptions); + return raftOptions; + } + + @Override + public String toString() { + return "RaftOptions{" + "maxByteCountPerRpc=" + this.maxByteCountPerRpc + ", fileCheckHole=" + + this.fileCheckHole + ", maxEntriesSize=" + this.maxEntriesSize + ", maxBodySize=" + this.maxBodySize + + ", maxAppendBufferSize=" + this.maxAppendBufferSize + ", maxElectionDelayMs=" + + this.maxElectionDelayMs + ", electionHeartbeatFactor=" + this.electionHeartbeatFactor + + ", applyBatch=" + this.applyBatch + ", sync=" + this.sync + ", syncMeta=" + this.syncMeta + + ", openStatistics=" + this.openStatistics + ", replicatorPipeline=" + this.replicatorPipeline + + ", maxReplicatorInflightMsgs=" + this.maxReplicatorInflightMsgs + ", disruptorBufferSize=" + + this.disruptorBufferSize + ", disruptorPublishEventWaitTimeoutSecs=" + + this.disruptorPublishEventWaitTimeoutSecs + ", enableLogEntryChecksum=" + this.enableLogEntryChecksum + + ", readOnlyOptions=" + this.readOnlyOptions + '}'; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyOption.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyOption.java new file mode 100644 index 0000000..ac45bf8 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyOption.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +/** + * Read only options. + * + * @author dennis + */ +public enum ReadOnlyOption { + + // ReadOnlySafe guarantees the linearizability of the read only request by + // communicating with the quorum. It is the default and suggested option. + ReadOnlySafe, + // ReadOnlyLeaseBased ensures linearizability of the read only request by + // relying on the leader lease. It can be affected by clock drift. + // If the clock drift is unbounded, leader might keep the lease longer than it + // should (clock can move backward/pause without any bound). ReadIndex is not safe + // in that case. + ReadOnlyLeaseBased +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyServiceOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyServiceOptions.java new file mode 100644 index 0000000..74f932a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReadOnlyServiceOptions.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.core.NodeImpl; + +/** + * Read-Only service options. + * + * @author dennis + */ +public class ReadOnlyServiceOptions { + + private RaftOptions raftOptions; + private NodeImpl node; + private FSMCaller fsmCaller; + + public NodeImpl getNode() { + return node; + } + + public void setNode(NodeImpl node) { + this.node = node; + } + + public RaftOptions getRaftOptions() { + return raftOptions; + } + + public void setRaftOptions(RaftOptions raftOptions) { + this.raftOptions = raftOptions; + } + + public FSMCaller getFsmCaller() { + return fsmCaller; + } + + public void setFsmCaller(FSMCaller fsm) { + this.fsmCaller = fsm; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorGroupOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorGroupOptions.java new file mode 100644 index 0000000..25c81af --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorGroupOptions.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.core.BallotBox; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.core.Scheduler; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.SnapshotStorage; + +/** + * Replicator group options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 5:47:43 PM + */ +public class ReplicatorGroupOptions { + + private int heartbeatTimeoutMs; + private int electionTimeoutMs; + private LogManager logManager; + private BallotBox ballotBox; + private NodeImpl node; + private SnapshotStorage snapshotStorage; + private RaftClientService raftRpcClientService; + private RaftOptions raftOptions; + private Scheduler timerManager; + + public Scheduler getTimerManager() { + return this.timerManager; + } + + public void setTimerManager(Scheduler timerManager) { + this.timerManager = timerManager; + } + + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + public void setRaftOptions(RaftOptions raftOptions) { + this.raftOptions = raftOptions; + } + + public RaftClientService getRaftRpcClientService() { + return this.raftRpcClientService; + } + + public void setRaftRpcClientService(RaftClientService raftRpcService) { + this.raftRpcClientService = raftRpcService; + } + + public int getHeartbeatTimeoutMs() { + return this.heartbeatTimeoutMs; + } + + public void setHeartbeatTimeoutMs(int heartbeatTimeoutMs) { + this.heartbeatTimeoutMs = heartbeatTimeoutMs; + } + + public int getElectionTimeoutMs() { + return this.electionTimeoutMs; + } + + public void setElectionTimeoutMs(int electionTimeoutMs) { + this.electionTimeoutMs = electionTimeoutMs; + } + + public LogManager getLogManager() { + return this.logManager; + } + + public void setLogManager(LogManager logManager) { + this.logManager = logManager; + } + + public BallotBox getBallotBox() { + return this.ballotBox; + } + + public void setBallotBox(BallotBox ballotBox) { + this.ballotBox = ballotBox; + } + + public NodeImpl getNode() { + return this.node; + } + + public void setNode(NodeImpl node) { + this.node = node; + } + + public SnapshotStorage getSnapshotStorage() { + return this.snapshotStorage; + } + + public void setSnapshotStorage(SnapshotStorage snapshotStorage) { + this.snapshotStorage = snapshotStorage; + } + + @Override + public String toString() { + return "ReplicatorGroupOptions{" + "heartbeatTimeoutMs=" + heartbeatTimeoutMs + ", electionTimeoutMs=" + + electionTimeoutMs + ", logManager=" + logManager + ", ballotBox=" + ballotBox + ", node=" + node + + ", snapshotStorage=" + snapshotStorage + ", raftRpcClientService=" + raftRpcClientService + + ", raftOptions=" + raftOptions + ", timerManager=" + timerManager + '}'; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorOptions.java new file mode 100644 index 0000000..c19fa30 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/ReplicatorOptions.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.core.BallotBox; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.core.ReplicatorType; +import com.alipay.sofa.jraft.core.Scheduler; +import com.alipay.sofa.jraft.core.TimerManager; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.SnapshotStorage; +import com.alipay.sofa.jraft.util.Copiable; + +/** + * Replicator options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:59:24 PM + */ +public class ReplicatorOptions implements Copiable { + + private int dynamicHeartBeatTimeoutMs; + private int electionTimeoutMs; + private String groupId; + private PeerId serverId; + private PeerId peerId; + private LogManager logManager; + private BallotBox ballotBox; + private NodeImpl node; + private long term; + private SnapshotStorage snapshotStorage; + private RaftClientService raftRpcService; + private Scheduler timerManager; + private ReplicatorType replicatorType; + + public ReplicatorOptions() { + super(); + } + + public ReplicatorOptions(final ReplicatorType replicatorType, final int dynamicHeartBeatTimeoutMs, + final int electionTimeoutMs, final String groupId, final PeerId serverId, + final PeerId peerId, final LogManager logManager, final BallotBox ballotBox, + final NodeImpl node, final long term, final SnapshotStorage snapshotStorage, + final RaftClientService raftRpcService, final TimerManager timerManager) { + super(); + this.replicatorType = replicatorType; + this.dynamicHeartBeatTimeoutMs = dynamicHeartBeatTimeoutMs; + this.electionTimeoutMs = electionTimeoutMs; + this.groupId = groupId; + this.serverId = serverId; + if (peerId != null) { + this.peerId = peerId.copy(); + } else { + this.peerId = null; + } + this.logManager = logManager; + this.ballotBox = ballotBox; + this.node = node; + this.term = term; + this.snapshotStorage = snapshotStorage; + this.raftRpcService = raftRpcService; + this.timerManager = timerManager; + } + + public final ReplicatorType getReplicatorType() { + return this.replicatorType; + } + + public void setReplicatorType(final ReplicatorType replicatorType) { + this.replicatorType = replicatorType; + } + + public RaftClientService getRaftRpcService() { + return this.raftRpcService; + } + + public void setRaftRpcService(final RaftClientService raftRpcService) { + this.raftRpcService = raftRpcService; + } + + @Override + public ReplicatorOptions copy() { + final ReplicatorOptions replicatorOptions = new ReplicatorOptions(); + replicatorOptions.setDynamicHeartBeatTimeoutMs(this.dynamicHeartBeatTimeoutMs); + replicatorOptions.setReplicatorType(this.replicatorType); + replicatorOptions.setElectionTimeoutMs(this.electionTimeoutMs); + replicatorOptions.setGroupId(this.groupId); + replicatorOptions.setServerId(this.serverId); + replicatorOptions.setPeerId(this.peerId); + replicatorOptions.setLogManager(this.logManager); + replicatorOptions.setBallotBox(this.ballotBox); + replicatorOptions.setNode(this.node); + replicatorOptions.setTerm(this.term); + replicatorOptions.setSnapshotStorage(this.snapshotStorage); + replicatorOptions.setRaftRpcService(this.raftRpcService); + replicatorOptions.setTimerManager(this.timerManager); + return replicatorOptions; + } + + public Scheduler getTimerManager() { + return this.timerManager; + } + + public void setTimerManager(final Scheduler timerManager) { + this.timerManager = timerManager; + } + + public PeerId getPeerId() { + return this.peerId; + } + + public void setPeerId(final PeerId peerId) { + if (peerId != null) { + this.peerId = peerId.copy(); + } else { + this.peerId = null; + } + } + + public int getDynamicHeartBeatTimeoutMs() { + return this.dynamicHeartBeatTimeoutMs; + } + + public void setDynamicHeartBeatTimeoutMs(final int dynamicHeartBeatTimeoutMs) { + this.dynamicHeartBeatTimeoutMs = dynamicHeartBeatTimeoutMs; + } + + public int getElectionTimeoutMs() { + return this.electionTimeoutMs; + } + + public void setElectionTimeoutMs(final int electionTimeoutMs) { + this.electionTimeoutMs = electionTimeoutMs; + } + + public String getGroupId() { + return this.groupId; + } + + public void setGroupId(final String groupId) { + this.groupId = groupId; + } + + public PeerId getServerId() { + return this.serverId; + } + + public void setServerId(final PeerId serverId) { + this.serverId = serverId; + } + + public LogManager getLogManager() { + return this.logManager; + } + + public void setLogManager(final LogManager logManager) { + this.logManager = logManager; + } + + public BallotBox getBallotBox() { + return this.ballotBox; + } + + public void setBallotBox(final BallotBox ballotBox) { + this.ballotBox = ballotBox; + } + + public NodeImpl getNode() { + return this.node; + } + + public void setNode(final NodeImpl node) { + this.node = node; + } + + public long getTerm() { + return this.term; + } + + public void setTerm(final long term) { + this.term = term; + } + + public SnapshotStorage getSnapshotStorage() { + return this.snapshotStorage; + } + + public void setSnapshotStorage(final SnapshotStorage snapshotStorage) { + this.snapshotStorage = snapshotStorage; + } + + @Override + public String toString() { + return "ReplicatorOptions{" + "replicatorType=" + this.replicatorType + "dynamicHeartBeatTimeoutMs=" + + this.dynamicHeartBeatTimeoutMs + ", electionTimeoutMs=" + this.electionTimeoutMs + ", groupId='" + + this.groupId + '\'' + ", serverId=" + this.serverId + ", peerId=" + this.peerId + ", logManager=" + + this.logManager + ", ballotBox=" + this.ballotBox + ", node=" + this.node + ", term=" + this.term + + ", snapshotStorage=" + this.snapshotStorage + ", raftRpcService=" + this.raftRpcService + + ", timerManager=" + this.timerManager + '}'; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RpcOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RpcOptions.java new file mode 100644 index 0000000..0a9d2cf --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/RpcOptions.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.codahale.metrics.MetricRegistry; + +public class RpcOptions { + + /** + * Rpc connect timeout in milliseconds + * Default: 1000(1s) + */ + private int rpcConnectTimeoutMs = 1000; + + /** + * RPC request default timeout in milliseconds + * Default: 5000(5s) + */ + private int rpcDefaultTimeout = 5000; + + /** + * Install snapshot RPC request default timeout in milliseconds + * Default: 5 * 60 * 1000(5min) + */ + private int rpcInstallSnapshotTimeout = 5 * 60 * 1000; + + /** + * RPC process thread pool size + * Default: 80 + */ + private int rpcProcessorThreadPoolSize = 80; + + /** + * Whether to enable checksum for RPC. + * Default: false + */ + private boolean enableRpcChecksum = false; + + /** + * Metric registry for RPC services, user should not use this field. + */ + private MetricRegistry metricRegistry; + + public int getRpcConnectTimeoutMs() { + return this.rpcConnectTimeoutMs; + } + + public void setRpcConnectTimeoutMs(int rpcConnectTimeoutMs) { + this.rpcConnectTimeoutMs = rpcConnectTimeoutMs; + } + + public int getRpcDefaultTimeout() { + return this.rpcDefaultTimeout; + } + + public void setRpcDefaultTimeout(int rpcDefaultTimeout) { + this.rpcDefaultTimeout = rpcDefaultTimeout; + } + + public int getRpcInstallSnapshotTimeout() { + return rpcInstallSnapshotTimeout; + } + + public void setRpcInstallSnapshotTimeout(int rpcInstallSnapshotTimeout) { + this.rpcInstallSnapshotTimeout = rpcInstallSnapshotTimeout; + } + + public int getRpcProcessorThreadPoolSize() { + return this.rpcProcessorThreadPoolSize; + } + + public void setRpcProcessorThreadPoolSize(int rpcProcessorThreadPoolSize) { + this.rpcProcessorThreadPoolSize = rpcProcessorThreadPoolSize; + } + + public boolean isEnableRpcChecksum() { + return enableRpcChecksum; + } + + public void setEnableRpcChecksum(boolean enableRpcChecksum) { + this.enableRpcChecksum = enableRpcChecksum; + } + + public MetricRegistry getMetricRegistry() { + return metricRegistry; + } + + public void setMetricRegistry(MetricRegistry metricRegistry) { + this.metricRegistry = metricRegistry; + } + + @Override + public String toString() { + return "RpcOptions{" + "rpcConnectTimeoutMs=" + rpcConnectTimeoutMs + ", rpcDefaultTimeout=" + + rpcDefaultTimeout + ", rpcInstallSnapshotTimeout=" + rpcInstallSnapshotTimeout + + ", rpcProcessorThreadPoolSize=" + rpcProcessorThreadPoolSize + ", enableRpcChecksum=" + + enableRpcChecksum + ", metricRegistry=" + metricRegistry + '}'; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotCopierOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotCopierOptions.java new file mode 100644 index 0000000..220ca9f --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotCopierOptions.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.core.Scheduler; +import com.alipay.sofa.jraft.rpc.RaftClientService; + +/** + * Snapshot copier options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-17 2:38:22 PM + */ +public class SnapshotCopierOptions { + + private RaftClientService raftClientService; + private Scheduler timerManager; + private RaftOptions raftOptions; + private NodeOptions nodeOptions; + + public SnapshotCopierOptions() { + super(); + } + + public SnapshotCopierOptions(RaftClientService raftClientService, Scheduler timerManager, RaftOptions raftOptions, + NodeOptions nodeOptions) { + super(); + this.raftClientService = raftClientService; + this.timerManager = timerManager; + this.raftOptions = raftOptions; + this.nodeOptions = nodeOptions; + } + + public NodeOptions getNodeOptions() { + return this.nodeOptions; + } + + public void setNodeOptions(NodeOptions nodeOptions) { + this.nodeOptions = nodeOptions; + } + + public RaftClientService getRaftClientService() { + return this.raftClientService; + } + + public void setRaftClientService(RaftClientService raftClientService) { + this.raftClientService = raftClientService; + } + + public Scheduler getTimerManager() { + return this.timerManager; + } + + public void setTimerManager(Scheduler timerManager) { + this.timerManager = timerManager; + } + + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + public void setRaftOptions(RaftOptions raftOptions) { + this.raftOptions = raftOptions; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotExecutorOptions.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotExecutorOptions.java new file mode 100644 index 0000000..d8ef7c8 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/option/SnapshotExecutorOptions.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.option; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.SnapshotThrottle; +import com.alipay.sofa.jraft.util.Endpoint; + +/** + * Snapshot executor options. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 2:59:37 PM + */ +public class SnapshotExecutorOptions { + + // URI of SnapshotStorage + private String uri; + private FSMCaller fsmCaller; + private NodeImpl node; + private LogManager logManager; + private long initTerm; + private Endpoint addr; + private boolean filterBeforeCopyRemote; + private SnapshotThrottle snapshotThrottle; + + public SnapshotThrottle getSnapshotThrottle() { + return snapshotThrottle; + } + + public void setSnapshotThrottle(SnapshotThrottle snapshotThrottle) { + this.snapshotThrottle = snapshotThrottle; + } + + public String getUri() { + return this.uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public FSMCaller getFsmCaller() { + return this.fsmCaller; + } + + public void setFsmCaller(FSMCaller fsmCaller) { + this.fsmCaller = fsmCaller; + } + + public NodeImpl getNode() { + return this.node; + } + + public void setNode(NodeImpl node) { + this.node = node; + } + + public LogManager getLogManager() { + return this.logManager; + } + + public void setLogManager(LogManager logManager) { + this.logManager = logManager; + } + + public long getInitTerm() { + return this.initTerm; + } + + public void setInitTerm(long initTerm) { + this.initTerm = initTerm; + } + + public Endpoint getAddr() { + return this.addr; + } + + public void setAddr(Endpoint addr) { + this.addr = addr; + } + + public boolean isFilterBeforeCopyRemote() { + return this.filterBeforeCopyRemote; + } + + public void setFilterBeforeCopyRemote(boolean filterBeforeCopyRemote) { + this.filterBeforeCopyRemote = filterBeforeCopyRemote; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliClientService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliClientService.java new file mode 100644 index 0000000..aa2871c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliClientService.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Future; + +import com.alipay.sofa.jraft.util.Endpoint; +import com.google.protobuf.Message; + +/** + * Cli RPC client service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 11:15:13 AM + */ +public interface CliClientService extends ClientService { + + /** + * Adds a peer. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future addPeer(Endpoint endpoint, CliRequests.AddPeerRequest request, + RpcResponseClosure done); + + /** + * Removes a peer. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future removePeer(Endpoint endpoint, CliRequests.RemovePeerRequest request, + RpcResponseClosure done); + + /** + * Reset a peer. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future resetPeer(Endpoint endpoint, CliRequests.ResetPeerRequest request, + RpcResponseClosure done); + + /** + * Do a snapshot. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future snapshot(Endpoint endpoint, CliRequests.SnapshotRequest request, + RpcResponseClosure done); + + /** + * Change peers. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future changePeers(Endpoint endpoint, CliRequests.ChangePeersRequest request, + RpcResponseClosure done); + + /** + * Add learners + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + * @since 1.3.0 + */ + Future addLearners(Endpoint endpoint, CliRequests.AddLearnersRequest request, + RpcResponseClosure done); + + /** + * Remove learners + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + * @since 1.3.0 + */ + Future removeLearners(Endpoint endpoint, CliRequests.RemoveLearnersRequest request, + RpcResponseClosure done); + + /** + * Reset learners + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + * @since 1.3.0 + */ + Future resetLearners(Endpoint endpoint, CliRequests.ResetLearnersRequest request, + RpcResponseClosure done); + + /** + * Get the group leader. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future getLeader(Endpoint endpoint, CliRequests.GetLeaderRequest request, + RpcResponseClosure done); + + /** + * Transfer leadership to other peer. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future transferLeader(Endpoint endpoint, CliRequests.TransferLeaderRequest request, + RpcResponseClosure done); + + /** + * Get all peers of the replication group. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @return a future with result + */ + Future getPeers(Endpoint endpoint, CliRequests.GetPeersRequest request, + RpcResponseClosure done); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java new file mode 100644 index 0000000..46b1fc4 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/CliRequests.java @@ -0,0 +1,15793 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: cli.proto + +package com.alipay.sofa.jraft.rpc; + +public final class CliRequests { + private CliRequests() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public interface AddPeerRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AddPeerRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.AddPeerRequest} + */ + public static final class AddPeerRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AddPeerRequest) + AddPeerRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AddPeerRequest.newBuilder() to construct. + private AddPeerRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AddPeerRequest() { + groupId_ = ""; + leaderId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AddPeerRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AddPeerRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AddPeerRequest) + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AddPeerRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.AddPeerRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AddPeerRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AddPeerRequest(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface AddPeerResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AddPeerResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string old_peers = 1; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 1; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 1; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 1; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * repeated string new_peers = 2; + */ + java.util.List getNewPeersList(); + + /** + * repeated string new_peers = 2; + */ + int getNewPeersCount(); + + /** + * repeated string new_peers = 2; + */ + java.lang.String getNewPeers(int index); + + /** + * repeated string new_peers = 2; + */ + com.google.protobuf.ByteString getNewPeersBytes(int index); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.AddPeerResponse} + */ + public static final class AddPeerResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AddPeerResponse) + AddPeerResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AddPeerResponse.newBuilder() to construct. + private AddPeerResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AddPeerResponse() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AddPeerResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + oldPeers_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + newPeers_.add(bs); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.Builder.class); + } + + private int bitField0_; + public static final int OLD_PEERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int NEW_PEERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList newPeers_; + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_; + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, oldPeers_.getRaw(i)); + } + for (int i = 0; i < newPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, newPeers_.getRaw(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < newPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(newPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewPeersList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse) obj; + + boolean result = true; + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && getNewPeersList().equals(other.getNewPeersList()); + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (getNewPeersCount() > 0) { + hash = (37 * hash) + NEW_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getNewPeersList().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AddPeerResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AddPeerResponse) + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddPeerResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.oldPeers_ = oldPeers_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.newPeers_ = newPeers_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse.getDefaultInstance()) + return this; + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (!other.newPeers_.isEmpty()) { + if (newPeers_.isEmpty()) { + newPeers_ = other.newPeers_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureNewPeersIsMutable(); + newPeers_.addAll(other.newPeers_); + } + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 1; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewPeersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(newPeers_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_.getUnmodifiableView(); + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + /** + * repeated string new_peers = 2; + */ + public Builder setNewPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addAllNewPeers(java.lang.Iterable values) { + ensureNewPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newPeers_); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder clearNewPeers() { + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AddPeerResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.AddPeerResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AddPeerResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AddPeerResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface RemovePeerRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.RemovePeerRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.RemovePeerRequest} + */ + public static final class RemovePeerRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.RemovePeerRequest) + RemovePeerRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use RemovePeerRequest.newBuilder() to construct. + private RemovePeerRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private RemovePeerRequest() { + groupId_ = ""; + leaderId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private RemovePeerRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.RemovePeerRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.RemovePeerRequest) + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.RemovePeerRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.RemovePeerRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public RemovePeerRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RemovePeerRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface RemovePeerResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.RemovePeerResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string old_peers = 1; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 1; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 1; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 1; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * repeated string new_peers = 2; + */ + java.util.List getNewPeersList(); + + /** + * repeated string new_peers = 2; + */ + int getNewPeersCount(); + + /** + * repeated string new_peers = 2; + */ + java.lang.String getNewPeers(int index); + + /** + * repeated string new_peers = 2; + */ + com.google.protobuf.ByteString getNewPeersBytes(int index); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.RemovePeerResponse} + */ + public static final class RemovePeerResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.RemovePeerResponse) + RemovePeerResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use RemovePeerResponse.newBuilder() to construct. + private RemovePeerResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private RemovePeerResponse() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private RemovePeerResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + oldPeers_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + newPeers_.add(bs); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.Builder.class); + } + + private int bitField0_; + public static final int OLD_PEERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int NEW_PEERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList newPeers_; + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_; + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, oldPeers_.getRaw(i)); + } + for (int i = 0; i < newPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, newPeers_.getRaw(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < newPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(newPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewPeersList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse) obj; + + boolean result = true; + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && getNewPeersList().equals(other.getNewPeersList()); + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (getNewPeersCount() > 0) { + hash = (37 * hash) + NEW_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getNewPeersList().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.RemovePeerResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.RemovePeerResponse) + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemovePeerResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.oldPeers_ = oldPeers_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.newPeers_ = newPeers_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse.getDefaultInstance()) + return this; + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (!other.newPeers_.isEmpty()) { + if (newPeers_.isEmpty()) { + newPeers_ = other.newPeers_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureNewPeersIsMutable(); + newPeers_.addAll(other.newPeers_); + } + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 1; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewPeersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(newPeers_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_.getUnmodifiableView(); + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + /** + * repeated string new_peers = 2; + */ + public Builder setNewPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addAllNewPeers(java.lang.Iterable values) { + ensureNewPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newPeers_); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder clearNewPeers() { + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.RemovePeerResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.RemovePeerResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public RemovePeerResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RemovePeerResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ChangePeersRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ChangePeersRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * repeated string new_peers = 3; + */ + java.util.List getNewPeersList(); + + /** + * repeated string new_peers = 3; + */ + int getNewPeersCount(); + + /** + * repeated string new_peers = 3; + */ + java.lang.String getNewPeers(int index); + + /** + * repeated string new_peers = 3; + */ + com.google.protobuf.ByteString getNewPeersBytes(int index); + } + + /** + * Protobuf type {@code jraft.ChangePeersRequest} + */ + public static final class ChangePeersRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ChangePeersRequest) + ChangePeersRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ChangePeersRequest.newBuilder() to construct. + private ChangePeersRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ChangePeersRequest() { + groupId_ = ""; + leaderId_ = ""; + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ChangePeersRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + newPeers_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + newPeers_ = newPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int NEW_PEERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList newPeers_; + + /** + * repeated string new_peers = 3; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_; + } + + /** + * repeated string new_peers = 3; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 3; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 3; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + for (int i = 0; i < newPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, newPeers_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + { + int dataSize = 0; + for (int i = 0; i < newPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(newPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewPeersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && getNewPeersList().equals(other.getNewPeersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (getNewPeersCount() > 0) { + hash = (37 * hash) + NEW_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getNewPeersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ChangePeersRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ChangePeersRequest) + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + newPeers_ = newPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.newPeers_ = newPeers_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (!other.newPeers_.isEmpty()) { + if (newPeers_.isEmpty()) { + newPeers_ = other.newPeers_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureNewPeersIsMutable(); + newPeers_.addAll(other.newPeers_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewPeersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(newPeers_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string new_peers = 3; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_.getUnmodifiableView(); + } + + /** + * repeated string new_peers = 3; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 3; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 3; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + /** + * repeated string new_peers = 3; + */ + public Builder setNewPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 3; + */ + public Builder addNewPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 3; + */ + public Builder addAllNewPeers(java.lang.Iterable values) { + ensureNewPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newPeers_); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 3; + */ + public Builder clearNewPeers() { + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 3; + */ + public Builder addNewPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ChangePeersRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.ChangePeersRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ChangePeersRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ChangePeersRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ChangePeersResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ChangePeersResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string old_peers = 1; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 1; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 1; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 1; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * repeated string new_peers = 2; + */ + java.util.List getNewPeersList(); + + /** + * repeated string new_peers = 2; + */ + int getNewPeersCount(); + + /** + * repeated string new_peers = 2; + */ + java.lang.String getNewPeers(int index); + + /** + * repeated string new_peers = 2; + */ + com.google.protobuf.ByteString getNewPeersBytes(int index); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.ChangePeersResponse} + */ + public static final class ChangePeersResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ChangePeersResponse) + ChangePeersResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ChangePeersResponse.newBuilder() to construct. + private ChangePeersResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ChangePeersResponse() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ChangePeersResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + oldPeers_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + newPeers_.add(bs); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.Builder.class); + } + + private int bitField0_; + public static final int OLD_PEERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int NEW_PEERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList newPeers_; + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_; + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, oldPeers_.getRaw(i)); + } + for (int i = 0; i < newPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, newPeers_.getRaw(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < newPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(newPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewPeersList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse) obj; + + boolean result = true; + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && getNewPeersList().equals(other.getNewPeersList()); + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (getNewPeersCount() > 0) { + hash = (37 * hash) + NEW_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getNewPeersList().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ChangePeersResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ChangePeersResponse) + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ChangePeersResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.oldPeers_ = oldPeers_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = newPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.newPeers_ = newPeers_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse.getDefaultInstance()) + return this; + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (!other.newPeers_.isEmpty()) { + if (newPeers_.isEmpty()) { + newPeers_ = other.newPeers_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureNewPeersIsMutable(); + newPeers_.addAll(other.newPeers_); + } + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 1; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 1; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 1; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 1; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 1; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewPeersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(newPeers_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_.getUnmodifiableView(); + } + + /** + * repeated string new_peers = 2; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 2; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 2; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + /** + * repeated string new_peers = 2; + */ + public Builder setNewPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addAllNewPeers(java.lang.Iterable values) { + ensureNewPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newPeers_); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder clearNewPeers() { + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 2; + */ + public Builder addNewPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ChangePeersResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.ChangePeersResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ChangePeersResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ChangePeersResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface SnapshotRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.SnapshotRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * optional string peer_id = 2; + */ + boolean hasPeerId(); + + /** + * optional string peer_id = 2; + */ + java.lang.String getPeerId(); + + /** + * optional string peer_id = 2; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.SnapshotRequest} + */ + public static final class SnapshotRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.SnapshotRequest) + SnapshotRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use SnapshotRequest.newBuilder() to construct. + private SnapshotRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private SnapshotRequest() { + groupId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private SnapshotRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_SnapshotRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_SnapshotRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object peerId_; + + /** + * optional string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * optional string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.SnapshotRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.SnapshotRequest) + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_SnapshotRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_SnapshotRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_SnapshotRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000002; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * optional string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string peer_id = 2; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + /** + * optional string peer_id = 2; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * optional string peer_id = 2; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.SnapshotRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.SnapshotRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public SnapshotRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new SnapshotRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ResetPeerRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ResetPeerRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string peer_id = 2; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 2; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 2; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + + /** + * repeated string old_peers = 3; + */ + java.util.List getOldPeersList(); + + /** + * repeated string old_peers = 3; + */ + int getOldPeersCount(); + + /** + * repeated string old_peers = 3; + */ + java.lang.String getOldPeers(int index); + + /** + * repeated string old_peers = 3; + */ + com.google.protobuf.ByteString getOldPeersBytes(int index); + + /** + * repeated string new_peers = 4; + */ + java.util.List getNewPeersList(); + + /** + * repeated string new_peers = 4; + */ + int getNewPeersCount(); + + /** + * repeated string new_peers = 4; + */ + java.lang.String getNewPeers(int index); + + /** + * repeated string new_peers = 4; + */ + com.google.protobuf.ByteString getNewPeersBytes(int index); + } + + /** + * Protobuf type {@code jraft.ResetPeerRequest} + */ + public static final class ResetPeerRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ResetPeerRequest) + ResetPeerRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ResetPeerRequest.newBuilder() to construct. + private ResetPeerRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ResetPeerRequest() { + groupId_ = ""; + peerId_ = ""; + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ResetPeerRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + peerId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + oldPeers_.add(bs); + break; + } + case 34: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000008; + } + newPeers_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000008) == 0x00000008)) { + newPeers_ = newPeers_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetPeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetPeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int OLD_PEERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList oldPeers_; + + /** + * repeated string old_peers = 3; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_; + } + + /** + * repeated string old_peers = 3; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 3; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 3; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + public static final int NEW_PEERS_FIELD_NUMBER = 4; + private com.google.protobuf.LazyStringList newPeers_; + + /** + * repeated string new_peers = 4; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_; + } + + /** + * repeated string new_peers = 4; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 4; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 4; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, peerId_); + } + for (int i = 0; i < oldPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, oldPeers_.getRaw(i)); + } + for (int i = 0; i < newPeers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 4, newPeers_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, peerId_); + } + { + int dataSize = 0; + for (int i = 0; i < oldPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(oldPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < newPeers_.size(); i++) { + dataSize += computeStringSizeNoTag(newPeers_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewPeersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && getOldPeersList().equals(other.getOldPeersList()); + result = result && getNewPeersList().equals(other.getNewPeersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + if (getOldPeersCount() > 0) { + hash = (37 * hash) + OLD_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getOldPeersList().hashCode(); + } + if (getNewPeersCount() > 0) { + hash = (37 * hash) + NEW_PEERS_FIELD_NUMBER; + hash = (53 * hash) + getNewPeersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ResetPeerRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ResetPeerRequest) + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetPeerRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetPeerRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetPeerRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.peerId_ = peerId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + oldPeers_ = oldPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.oldPeers_ = oldPeers_; + if (((bitField0_ & 0x00000008) == 0x00000008)) { + newPeers_ = newPeers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000008); + } + result.newPeers_ = newPeers_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000002; + peerId_ = other.peerId_; + onChanged(); + } + if (!other.oldPeers_.isEmpty()) { + if (oldPeers_.isEmpty()) { + oldPeers_ = other.oldPeers_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureOldPeersIsMutable(); + oldPeers_.addAll(other.oldPeers_); + } + onChanged(); + } + if (!other.newPeers_.isEmpty()) { + if (newPeers_.isEmpty()) { + newPeers_ = other.newPeers_; + bitField0_ = (bitField0_ & ~0x00000008); + } else { + ensureNewPeersIsMutable(); + newPeers_.addAll(other.newPeers_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 2; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 2; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 2; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldPeersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + oldPeers_ = new com.google.protobuf.LazyStringArrayList(oldPeers_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string old_peers = 3; + */ + public com.google.protobuf.ProtocolStringList getOldPeersList() { + return oldPeers_.getUnmodifiableView(); + } + + /** + * repeated string old_peers = 3; + */ + public int getOldPeersCount() { + return oldPeers_.size(); + } + + /** + * repeated string old_peers = 3; + */ + public java.lang.String getOldPeers(int index) { + return oldPeers_.get(index); + } + + /** + * repeated string old_peers = 3; + */ + public com.google.protobuf.ByteString getOldPeersBytes(int index) { + return oldPeers_.getByteString(index); + } + + /** + * repeated string old_peers = 3; + */ + public Builder setOldPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 3; + */ + public Builder addOldPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 3; + */ + public Builder addAllOldPeers(java.lang.Iterable values) { + ensureOldPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldPeers_); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 3; + */ + public Builder clearOldPeers() { + oldPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string old_peers = 3; + */ + public Builder addOldPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldPeersIsMutable(); + oldPeers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewPeersIsMutable() { + if (!((bitField0_ & 0x00000008) == 0x00000008)) { + newPeers_ = new com.google.protobuf.LazyStringArrayList(newPeers_); + bitField0_ |= 0x00000008; + } + } + + /** + * repeated string new_peers = 4; + */ + public com.google.protobuf.ProtocolStringList getNewPeersList() { + return newPeers_.getUnmodifiableView(); + } + + /** + * repeated string new_peers = 4; + */ + public int getNewPeersCount() { + return newPeers_.size(); + } + + /** + * repeated string new_peers = 4; + */ + public java.lang.String getNewPeers(int index) { + return newPeers_.get(index); + } + + /** + * repeated string new_peers = 4; + */ + public com.google.protobuf.ByteString getNewPeersBytes(int index) { + return newPeers_.getByteString(index); + } + + /** + * repeated string new_peers = 4; + */ + public Builder setNewPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 4; + */ + public Builder addNewPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 4; + */ + public Builder addAllNewPeers(java.lang.Iterable values) { + ensureNewPeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newPeers_); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 4; + */ + public Builder clearNewPeers() { + newPeers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000008); + onChanged(); + return this; + } + + /** + * repeated string new_peers = 4; + */ + public Builder addNewPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewPeersIsMutable(); + newPeers_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ResetPeerRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.ResetPeerRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ResetPeerRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ResetPeerRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface TransferLeaderRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.TransferLeaderRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * optional string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * optional string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * optional string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.TransferLeaderRequest} + */ + public static final class TransferLeaderRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.TransferLeaderRequest) + TransferLeaderRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use TransferLeaderRequest.newBuilder() to construct. + private TransferLeaderRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private TransferLeaderRequest() { + groupId_ = ""; + leaderId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private TransferLeaderRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_TransferLeaderRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_TransferLeaderRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * optional string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * optional string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.TransferLeaderRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.TransferLeaderRequest) + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_TransferLeaderRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_TransferLeaderRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_TransferLeaderRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * optional string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * optional string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * optional string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.TransferLeaderRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.TransferLeaderRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public TransferLeaderRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new TransferLeaderRequest( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetLeaderRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetLeaderRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * optional string peer_id = 2; + */ + boolean hasPeerId(); + + /** + * optional string peer_id = 2; + */ + java.lang.String getPeerId(); + + /** + * optional string peer_id = 2; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.GetLeaderRequest} + */ + public static final class GetLeaderRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetLeaderRequest) + GetLeaderRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetLeaderRequest.newBuilder() to construct. + private GetLeaderRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetLeaderRequest() { + groupId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetLeaderRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object peerId_; + + /** + * optional string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * optional string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetLeaderRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetLeaderRequest) + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000002; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * optional string peer_id = 2; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string peer_id = 2; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string peer_id = 2; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string peer_id = 2; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + /** + * optional string peer_id = 2; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * optional string peer_id = 2; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetLeaderRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.GetLeaderRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetLeaderRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetLeaderRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetLeaderResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetLeaderResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required string leader_id = 1; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 1; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 1; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.GetLeaderResponse} + */ + public static final class GetLeaderResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetLeaderResponse) + GetLeaderResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetLeaderResponse.newBuilder() to construct. + private GetLeaderResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetLeaderResponse() { + leaderId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetLeaderResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + leaderId_ = bs; + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000002; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.Builder.class); + } + + private int bitField0_; + public static final int LEADER_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 1; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string leader_id = 1; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 1; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, leaderId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, leaderId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse) obj; + + boolean result = true; + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetLeaderResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetLeaderResponse) + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetLeaderResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.leaderId_ = leaderId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse.getDefaultInstance()) + return this; + if (other.hasLeaderId()) { + bitField0_ |= 0x00000001; + leaderId_ = other.leaderId_; + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasLeaderId()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 1; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string leader_id = 1; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 1; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 1; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 1; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 1; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + leaderId_ = value; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000002) == 0x00000002) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000002; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000002; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetLeaderResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.GetLeaderResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetLeaderResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetLeaderResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetPeersRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetPeersRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * optional string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * optional string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * optional string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * optional bool only_alive = 3 [default = false]; + */ + boolean hasOnlyAlive(); + + /** + * optional bool only_alive = 3 [default = false]; + */ + boolean getOnlyAlive(); + } + + /** + * Protobuf type {@code jraft.GetPeersRequest} + */ + public static final class GetPeersRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetPeersRequest) + GetPeersRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetPeersRequest.newBuilder() to construct. + private GetPeersRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetPeersRequest() { + groupId_ = ""; + leaderId_ = ""; + onlyAlive_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetPeersRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 24: { + bitField0_ |= 0x00000004; + onlyAlive_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * optional string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * optional string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int ONLY_ALIVE_FIELD_NUMBER = 3; + private boolean onlyAlive_; + + /** + * optional bool only_alive = 3 [default = false]; + */ + public boolean hasOnlyAlive() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional bool only_alive = 3 [default = false]; + */ + public boolean getOnlyAlive() { + return onlyAlive_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeBool(3, onlyAlive_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(3, onlyAlive_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && (hasOnlyAlive() == other.hasOnlyAlive()); + if (hasOnlyAlive()) { + result = result && (getOnlyAlive() == other.getOnlyAlive()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (hasOnlyAlive()) { + hash = (37 * hash) + ONLY_ALIVE_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getOnlyAlive()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetPeersRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetPeersRequest) + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + onlyAlive_ = false; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.onlyAlive_ = onlyAlive_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (other.hasOnlyAlive()) { + setOnlyAlive(other.getOnlyAlive()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * optional string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * optional string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * optional string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private boolean onlyAlive_; + + /** + * optional bool only_alive = 3 [default = false]; + */ + public boolean hasOnlyAlive() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional bool only_alive = 3 [default = false]; + */ + public boolean getOnlyAlive() { + return onlyAlive_; + } + + /** + * optional bool only_alive = 3 [default = false]; + */ + public Builder setOnlyAlive(boolean value) { + bitField0_ |= 0x00000004; + onlyAlive_ = value; + onChanged(); + return this; + } + + /** + * optional bool only_alive = 3 [default = false]; + */ + public Builder clearOnlyAlive() { + bitField0_ = (bitField0_ & ~0x00000004); + onlyAlive_ = false; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetPeersRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.GetPeersRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetPeersRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetPeersRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetPeersResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetPeersResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string peers = 1; + */ + java.util.List getPeersList(); + + /** + * repeated string peers = 1; + */ + int getPeersCount(); + + /** + * repeated string peers = 1; + */ + java.lang.String getPeers(int index); + + /** + * repeated string peers = 1; + */ + com.google.protobuf.ByteString getPeersBytes(int index); + + /** + * repeated string learners = 2; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 2; + */ + int getLearnersCount(); + + /** + * repeated string learners = 2; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 2; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.GetPeersResponse} + */ + public static final class GetPeersResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetPeersResponse) + GetPeersResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetPeersResponse.newBuilder() to construct. + private GetPeersResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetPeersResponse() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetPeersResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + peers_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + learners_.add(bs); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = peers_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + learners_ = learners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.Builder.class); + } + + private int bitField0_; + public static final int PEERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList peers_; + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_; + } + + /** + * repeated string peers = 1; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 1; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + public static final int LEARNERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 2; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 2; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 2; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 2; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < peers_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, peers_.getRaw(i)); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, learners_.getRaw(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < peers_.size(); i++) { + dataSize += computeStringSizeNoTag(peers_.getRaw(i)); + } + size += dataSize; + size += 1 * getPeersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse) obj; + + boolean result = true; + result = result && getPeersList().equals(other.getPeersList()); + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getPeersCount() > 0) { + hash = (37 * hash) + PEERS_FIELD_NUMBER; + hash = (53 * hash) + getPeersList().hashCode(); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetPeersResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetPeersResponse) + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_GetPeersResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = peers_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.peers_ = peers_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.learners_ = learners_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse.getDefaultInstance()) + return this; + if (!other.peers_.isEmpty()) { + if (peers_.isEmpty()) { + peers_ = other.peers_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensurePeersIsMutable(); + peers_.addAll(other.peers_); + } + onChanged(); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensurePeersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + peers_ = new com.google.protobuf.LazyStringArrayList(peers_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ProtocolStringList getPeersList() { + return peers_.getUnmodifiableView(); + } + + /** + * repeated string peers = 1; + */ + public int getPeersCount() { + return peers_.size(); + } + + /** + * repeated string peers = 1; + */ + public java.lang.String getPeers(int index) { + return peers_.get(index); + } + + /** + * repeated string peers = 1; + */ + public com.google.protobuf.ByteString getPeersBytes(int index) { + return peers_.getByteString(index); + } + + /** + * repeated string peers = 1; + */ + public Builder setPeers(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addPeers(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addAllPeers(java.lang.Iterable values) { + ensurePeersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, peers_); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder clearPeers() { + peers_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string peers = 1; + */ + public Builder addPeersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensurePeersIsMutable(); + peers_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string learners = 2; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 2; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 2; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 2; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 2; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 2; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 2; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 2; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string learners = 2; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetPeersResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.GetPeersResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetPeersResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetPeersResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface AddLearnersRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AddLearnersRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * repeated string learners = 3; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 3; + */ + int getLearnersCount(); + + /** + * repeated string learners = 3; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 3; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + } + + /** + * Protobuf type {@code jraft.AddLearnersRequest} + */ + public static final class AddLearnersRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AddLearnersRequest) + AddLearnersRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AddLearnersRequest.newBuilder() to construct. + private AddLearnersRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AddLearnersRequest() { + groupId_ = ""; + leaderId_ = ""; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AddLearnersRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + learners_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEARNERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, learners_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AddLearnersRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AddLearnersRequest) + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_AddLearnersRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.learners_ = learners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 3; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AddLearnersRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.AddLearnersRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AddLearnersRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AddLearnersRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface RemoveLearnersRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.RemoveLearnersRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * repeated string learners = 3; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 3; + */ + int getLearnersCount(); + + /** + * repeated string learners = 3; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 3; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + } + + /** + * Protobuf type {@code jraft.RemoveLearnersRequest} + */ + public static final class RemoveLearnersRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.RemoveLearnersRequest) + RemoveLearnersRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use RemoveLearnersRequest.newBuilder() to construct. + private RemoveLearnersRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private RemoveLearnersRequest() { + groupId_ = ""; + leaderId_ = ""; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private RemoveLearnersRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + learners_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemoveLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemoveLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEARNERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, learners_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.RemoveLearnersRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.RemoveLearnersRequest) + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemoveLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemoveLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_RemoveLearnersRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.learners_ = learners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 3; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.RemoveLearnersRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.RemoveLearnersRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public RemoveLearnersRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RemoveLearnersRequest( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ResetLearnersRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ResetLearnersRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string leader_id = 2; + */ + boolean hasLeaderId(); + + /** + * required string leader_id = 2; + */ + java.lang.String getLeaderId(); + + /** + * required string leader_id = 2; + */ + com.google.protobuf.ByteString getLeaderIdBytes(); + + /** + * repeated string learners = 3; + */ + java.util.List getLearnersList(); + + /** + * repeated string learners = 3; + */ + int getLearnersCount(); + + /** + * repeated string learners = 3; + */ + java.lang.String getLearners(int index); + + /** + * repeated string learners = 3; + */ + com.google.protobuf.ByteString getLearnersBytes(int index); + } + + /** + * Protobuf type {@code jraft.ResetLearnersRequest} + */ + public static final class ResetLearnersRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ResetLearnersRequest) + ResetLearnersRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ResetLearnersRequest.newBuilder() to construct. + private ResetLearnersRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ResetLearnersRequest() { + groupId_ = ""; + leaderId_ = ""; + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ResetLearnersRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + leaderId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000004; + } + learners_.add(bs); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEADER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object leaderId_; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int LEARNERS_FIELD_NUMBER = 3; + private com.google.protobuf.LazyStringList learners_; + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_; + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLeaderId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, leaderId_); + } + for (int i = 0; i < learners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, learners_.getRaw(i)); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, leaderId_); + } + { + int dataSize = 0; + for (int i = 0; i < learners_.size(); i++) { + dataSize += computeStringSizeNoTag(learners_.getRaw(i)); + } + size += dataSize; + size += 1 * getLearnersList().size(); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest other = (com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasLeaderId() == other.hasLeaderId()); + if (hasLeaderId()) { + result = result && getLeaderId().equals(other.getLeaderId()); + } + result = result && getLearnersList().equals(other.getLearnersList()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasLeaderId()) { + hash = (37 * hash) + LEADER_ID_FIELD_NUMBER; + hash = (53 * hash) + getLeaderId().hashCode(); + } + if (getLearnersCount() > 0) { + hash = (37 * hash) + LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getLearnersList().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ResetLearnersRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ResetLearnersRequest) + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetLearnersRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetLearnersRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.class, + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + leaderId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_ResetLearnersRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest build() { + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest result = new com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.leaderId_ = leaderId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = learners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.learners_ = learners_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasLeaderId()) { + bitField0_ |= 0x00000002; + leaderId_ = other.leaderId_; + onChanged(); + } + if (!other.learners_.isEmpty()) { + if (learners_.isEmpty()) { + learners_ = other.learners_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureLearnersIsMutable(); + learners_.addAll(other.learners_); + } + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasLeaderId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object leaderId_ = ""; + + /** + * required string leader_id = 2; + */ + public boolean hasLeaderId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string leader_id = 2; + */ + public java.lang.String getLeaderId() { + java.lang.Object ref = leaderId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + leaderId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string leader_id = 2; + */ + public com.google.protobuf.ByteString getLeaderIdBytes() { + java.lang.Object ref = leaderId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + leaderId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder clearLeaderId() { + bitField0_ = (bitField0_ & ~0x00000002); + leaderId_ = getDefaultInstance().getLeaderId(); + onChanged(); + return this; + } + + /** + * required string leader_id = 2; + */ + public Builder setLeaderIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + leaderId_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureLearnersIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + learners_ = new com.google.protobuf.LazyStringArrayList(learners_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ProtocolStringList getLearnersList() { + return learners_.getUnmodifiableView(); + } + + /** + * repeated string learners = 3; + */ + public int getLearnersCount() { + return learners_.size(); + } + + /** + * repeated string learners = 3; + */ + public java.lang.String getLearners(int index) { + return learners_.get(index); + } + + /** + * repeated string learners = 3; + */ + public com.google.protobuf.ByteString getLearnersBytes(int index) { + return learners_.getByteString(index); + } + + /** + * repeated string learners = 3; + */ + public Builder setLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addAllLearners(java.lang.Iterable values) { + ensureLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, learners_); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder clearLearners() { + learners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + /** + * repeated string learners = 3; + */ + public Builder addLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureLearnersIsMutable(); + learners_.add(value); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ResetLearnersRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.ResetLearnersRequest) + private static final com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ResetLearnersRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ResetLearnersRequest( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface LearnersOpResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.LearnersOpResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * repeated string old_learners = 1; + */ + java.util.List getOldLearnersList(); + + /** + * repeated string old_learners = 1; + */ + int getOldLearnersCount(); + + /** + * repeated string old_learners = 1; + */ + java.lang.String getOldLearners(int index); + + /** + * repeated string old_learners = 1; + */ + com.google.protobuf.ByteString getOldLearnersBytes(int index); + + /** + * repeated string new_learners = 2; + */ + java.util.List getNewLearnersList(); + + /** + * repeated string new_learners = 2; + */ + int getNewLearnersCount(); + + /** + * repeated string new_learners = 2; + */ + java.lang.String getNewLearners(int index); + + /** + * repeated string new_learners = 2; + */ + com.google.protobuf.ByteString getNewLearnersBytes(int index); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.LearnersOpResponse} + */ + public static final class LearnersOpResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.LearnersOpResponse) + LearnersOpResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use LearnersOpResponse.newBuilder() to construct. + private LearnersOpResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private LearnersOpResponse() { + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + newLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private LearnersOpResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000001; + } + oldLearners_.add(bs); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + if (!((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newLearners_ = new com.google.protobuf.LazyStringArrayList(); + mutable_bitField0_ |= 0x00000002; + } + newLearners_.add(bs); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000001; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + } + if (((mutable_bitField0_ & 0x00000002) == 0x00000002)) { + newLearners_ = newLearners_.getUnmodifiableView(); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_LearnersOpResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_LearnersOpResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.Builder.class); + } + + private int bitField0_; + public static final int OLD_LEARNERS_FIELD_NUMBER = 1; + private com.google.protobuf.LazyStringList oldLearners_; + + /** + * repeated string old_learners = 1; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_; + } + + /** + * repeated string old_learners = 1; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 1; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 1; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + public static final int NEW_LEARNERS_FIELD_NUMBER = 2; + private com.google.protobuf.LazyStringList newLearners_; + + /** + * repeated string new_learners = 2; + */ + public com.google.protobuf.ProtocolStringList getNewLearnersList() { + return newLearners_; + } + + /** + * repeated string new_learners = 2; + */ + public int getNewLearnersCount() { + return newLearners_.size(); + } + + /** + * repeated string new_learners = 2; + */ + public java.lang.String getNewLearners(int index) { + return newLearners_.get(index); + } + + /** + * repeated string new_learners = 2; + */ + public com.google.protobuf.ByteString getNewLearnersBytes(int index) { + return newLearners_.getByteString(index); + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + for (int i = 0; i < oldLearners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, oldLearners_.getRaw(i)); + } + for (int i = 0; i < newLearners_.size(); i++) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, newLearners_.getRaw(i)); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + { + int dataSize = 0; + for (int i = 0; i < oldLearners_.size(); i++) { + dataSize += computeStringSizeNoTag(oldLearners_.getRaw(i)); + } + size += dataSize; + size += 1 * getOldLearnersList().size(); + } + { + int dataSize = 0; + for (int i = 0; i < newLearners_.size(); i++) { + dataSize += computeStringSizeNoTag(newLearners_.getRaw(i)); + } + size += dataSize; + size += 1 * getNewLearnersList().size(); + } + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse other = (com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse) obj; + + boolean result = true; + result = result && getOldLearnersList().equals(other.getOldLearnersList()); + result = result && getNewLearnersList().equals(other.getNewLearnersList()); + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (getOldLearnersCount() > 0) { + hash = (37 * hash) + OLD_LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getOldLearnersList().hashCode(); + } + if (getNewLearnersCount() > 0) { + hash = (37 * hash) + NEW_LEARNERS_FIELD_NUMBER; + hash = (53 * hash) + getNewLearnersList().hashCode(); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.LearnersOpResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.LearnersOpResponse) + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_LearnersOpResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_LearnersOpResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.class, + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + newLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.internal_static_jraft_LearnersOpResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse build() { + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse buildPartial() { + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse result = new com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + oldLearners_ = oldLearners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000001); + } + result.oldLearners_ = oldLearners_; + if (((bitField0_ & 0x00000002) == 0x00000002)) { + newLearners_ = newLearners_.getUnmodifiableView(); + bitField0_ = (bitField0_ & ~0x00000002); + } + result.newLearners_ = newLearners_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000001; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse other) { + if (other == com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse.getDefaultInstance()) + return this; + if (!other.oldLearners_.isEmpty()) { + if (oldLearners_.isEmpty()) { + oldLearners_ = other.oldLearners_; + bitField0_ = (bitField0_ & ~0x00000001); + } else { + ensureOldLearnersIsMutable(); + oldLearners_.addAll(other.oldLearners_); + } + onChanged(); + } + if (!other.newLearners_.isEmpty()) { + if (newLearners_.isEmpty()) { + newLearners_ = other.newLearners_; + bitField0_ = (bitField0_ & ~0x00000002); + } else { + ensureNewLearnersIsMutable(); + newLearners_.addAll(other.newLearners_); + } + onChanged(); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private com.google.protobuf.LazyStringList oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureOldLearnersIsMutable() { + if (!((bitField0_ & 0x00000001) == 0x00000001)) { + oldLearners_ = new com.google.protobuf.LazyStringArrayList(oldLearners_); + bitField0_ |= 0x00000001; + } + } + + /** + * repeated string old_learners = 1; + */ + public com.google.protobuf.ProtocolStringList getOldLearnersList() { + return oldLearners_.getUnmodifiableView(); + } + + /** + * repeated string old_learners = 1; + */ + public int getOldLearnersCount() { + return oldLearners_.size(); + } + + /** + * repeated string old_learners = 1; + */ + public java.lang.String getOldLearners(int index) { + return oldLearners_.get(index); + } + + /** + * repeated string old_learners = 1; + */ + public com.google.protobuf.ByteString getOldLearnersBytes(int index) { + return oldLearners_.getByteString(index); + } + + /** + * repeated string old_learners = 1; + */ + public Builder setOldLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 1; + */ + public Builder addOldLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 1; + */ + public Builder addAllOldLearners(java.lang.Iterable values) { + ensureOldLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, oldLearners_); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 1; + */ + public Builder clearOldLearners() { + oldLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000001); + onChanged(); + return this; + } + + /** + * repeated string old_learners = 1; + */ + public Builder addOldLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureOldLearnersIsMutable(); + oldLearners_.add(value); + onChanged(); + return this; + } + + private com.google.protobuf.LazyStringList newLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + + private void ensureNewLearnersIsMutable() { + if (!((bitField0_ & 0x00000002) == 0x00000002)) { + newLearners_ = new com.google.protobuf.LazyStringArrayList(newLearners_); + bitField0_ |= 0x00000002; + } + } + + /** + * repeated string new_learners = 2; + */ + public com.google.protobuf.ProtocolStringList getNewLearnersList() { + return newLearners_.getUnmodifiableView(); + } + + /** + * repeated string new_learners = 2; + */ + public int getNewLearnersCount() { + return newLearners_.size(); + } + + /** + * repeated string new_learners = 2; + */ + public java.lang.String getNewLearners(int index) { + return newLearners_.get(index); + } + + /** + * repeated string new_learners = 2; + */ + public com.google.protobuf.ByteString getNewLearnersBytes(int index) { + return newLearners_.getByteString(index); + } + + /** + * repeated string new_learners = 2; + */ + public Builder setNewLearners(int index, java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewLearnersIsMutable(); + newLearners_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated string new_learners = 2; + */ + public Builder addNewLearners(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewLearnersIsMutable(); + newLearners_.add(value); + onChanged(); + return this; + } + + /** + * repeated string new_learners = 2; + */ + public Builder addAllNewLearners(java.lang.Iterable values) { + ensureNewLearnersIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, newLearners_); + onChanged(); + return this; + } + + /** + * repeated string new_learners = 2; + */ + public Builder clearNewLearners() { + newLearners_ = com.google.protobuf.LazyStringArrayList.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + onChanged(); + return this; + } + + /** + * repeated string new_learners = 2; + */ + public Builder addNewLearnersBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureNewLearnersIsMutable(); + newLearners_.add(value); + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.LearnersOpResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.LearnersOpResponse) + private static final com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse(); + } + + public static com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public LearnersOpResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new LearnersOpResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AddPeerRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AddPeerRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AddPeerResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AddPeerResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_RemovePeerRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_RemovePeerRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_RemovePeerResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_RemovePeerResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ChangePeersRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ChangePeersRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ChangePeersResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ChangePeersResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_SnapshotRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_SnapshotRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ResetPeerRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ResetPeerRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_TransferLeaderRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_TransferLeaderRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetLeaderRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetLeaderRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetLeaderResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetLeaderResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetPeersRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetPeersRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetPeersResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetPeersResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AddLearnersRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AddLearnersRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_RemoveLearnersRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_RemoveLearnersRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ResetLearnersRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ResetLearnersRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_LearnersOpResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_LearnersOpResponse_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\tcli.proto\022\005jraft\032\trpc.proto\"F\n\016AddPeer" + + "Request\022\020\n\010group_id\030\001 \002(\t\022\021\n\tleader_id\030\002" + + " \002(\t\022\017\n\007peer_id\030\003 \002(\t\"d\n\017AddPeerResponse" + + "\022\021\n\told_peers\030\001 \003(\t\022\021\n\tnew_peers\030\002 \003(\t\022+" + + "\n\rerrorResponse\030c \001(\0132\024.jraft.ErrorRespo" + + "nse\"I\n\021RemovePeerRequest\022\020\n\010group_id\030\001 \002" + + "(\t\022\021\n\tleader_id\030\002 \002(\t\022\017\n\007peer_id\030\003 \002(\t\"g" + + "\n\022RemovePeerResponse\022\021\n\told_peers\030\001 \003(\t\022" + + "\021\n\tnew_peers\030\002 \003(\t\022+\n\rerrorResponse\030c \001(" + + "\0132\024.jraft.ErrorResponse\"L\n\022ChangePeersRe" + + "quest\022\020\n\010group_id\030\001 \002(\t\022\021\n\tleader_id\030\002 \002" + + "(\t\022\021\n\tnew_peers\030\003 \003(\t\"h\n\023ChangePeersResp" + + "onse\022\021\n\told_peers\030\001 \003(\t\022\021\n\tnew_peers\030\002 \003" + + "(\t\022+\n\rerrorResponse\030c \001(\0132\024.jraft.ErrorR" + + "esponse\"4\n\017SnapshotRequest\022\020\n\010group_id\030\001" + + " \002(\t\022\017\n\007peer_id\030\002 \001(\t\"[\n\020ResetPeerReques" + + "t\022\020\n\010group_id\030\001 \002(\t\022\017\n\007peer_id\030\002 \002(\t\022\021\n\t" + + "old_peers\030\003 \003(\t\022\021\n\tnew_peers\030\004 \003(\t\"M\n\025Tr" + + "ansferLeaderRequest\022\020\n\010group_id\030\001 \002(\t\022\021\n" + + "\tleader_id\030\002 \002(\t\022\017\n\007peer_id\030\003 \001(\t\"5\n\020Get" + + "LeaderRequest\022\020\n\010group_id\030\001 \002(\t\022\017\n\007peer_" + + "id\030\002 \001(\t\"S\n\021GetLeaderResponse\022\021\n\tleader_" + + "id\030\001 \002(\t\022+\n\rerrorResponse\030c \001(\0132\024.jraft." + + "ErrorResponse\"Q\n\017GetPeersRequest\022\020\n\010grou" + + "p_id\030\001 \002(\t\022\021\n\tleader_id\030\002 \001(\t\022\031\n\nonly_al" + + "ive\030\003 \001(\010:\005false\"`\n\020GetPeersResponse\022\r\n\005" + + "peers\030\001 \003(\t\022\020\n\010learners\030\002 \003(\t\022+\n\rerrorRe" + + "sponse\030c \001(\0132\024.jraft.ErrorResponse\"K\n\022Ad" + + "dLearnersRequest\022\020\n\010group_id\030\001 \002(\t\022\021\n\tle" + + "ader_id\030\002 \002(\t\022\020\n\010learners\030\003 \003(\t\"N\n\025Remov" + + "eLearnersRequest\022\020\n\010group_id\030\001 \002(\t\022\021\n\tle" + + "ader_id\030\002 \002(\t\022\020\n\010learners\030\003 \003(\t\"M\n\024Reset" + + "LearnersRequest\022\020\n\010group_id\030\001 \002(\t\022\021\n\tlea" + + "der_id\030\002 \002(\t\022\020\n\010learners\030\003 \003(\t\"m\n\022Learne" + + "rsOpResponse\022\024\n\014old_learners\030\001 \003(\t\022\024\n\014ne" + + "w_learners\030\002 \003(\t\022+\n\rerrorResponse\030c \001(\0132" + + "\024.jraft.ErrorResponseB(\n\031com.alipay.sofa" + + ".jraft.rpcB\013CliRequests" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { com.alipay.sofa.jraft.rpc.RpcRequests + .getDescriptor(), }, assigner); + internal_static_jraft_AddPeerRequest_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_AddPeerRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AddPeerRequest_descriptor, + new java.lang.String[] { "GroupId", "LeaderId", "PeerId", }); + internal_static_jraft_AddPeerResponse_descriptor = getDescriptor().getMessageTypes().get(1); + internal_static_jraft_AddPeerResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AddPeerResponse_descriptor, new java.lang.String[] { "OldPeers", "NewPeers", + "ErrorResponse", }); + internal_static_jraft_RemovePeerRequest_descriptor = getDescriptor().getMessageTypes().get(2); + internal_static_jraft_RemovePeerRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_RemovePeerRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "PeerId", }); + internal_static_jraft_RemovePeerResponse_descriptor = getDescriptor().getMessageTypes().get(3); + internal_static_jraft_RemovePeerResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_RemovePeerResponse_descriptor, new java.lang.String[] { "OldPeers", "NewPeers", + "ErrorResponse", }); + internal_static_jraft_ChangePeersRequest_descriptor = getDescriptor().getMessageTypes().get(4); + internal_static_jraft_ChangePeersRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ChangePeersRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "NewPeers", }); + internal_static_jraft_ChangePeersResponse_descriptor = getDescriptor().getMessageTypes().get(5); + internal_static_jraft_ChangePeersResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ChangePeersResponse_descriptor, new java.lang.String[] { "OldPeers", "NewPeers", + "ErrorResponse", }); + internal_static_jraft_SnapshotRequest_descriptor = getDescriptor().getMessageTypes().get(6); + internal_static_jraft_SnapshotRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_SnapshotRequest_descriptor, new java.lang.String[] { "GroupId", "PeerId", }); + internal_static_jraft_ResetPeerRequest_descriptor = getDescriptor().getMessageTypes().get(7); + internal_static_jraft_ResetPeerRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ResetPeerRequest_descriptor, new java.lang.String[] { "GroupId", "PeerId", + "OldPeers", "NewPeers", }); + internal_static_jraft_TransferLeaderRequest_descriptor = getDescriptor().getMessageTypes().get(8); + internal_static_jraft_TransferLeaderRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_TransferLeaderRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "PeerId", }); + internal_static_jraft_GetLeaderRequest_descriptor = getDescriptor().getMessageTypes().get(9); + internal_static_jraft_GetLeaderRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetLeaderRequest_descriptor, new java.lang.String[] { "GroupId", "PeerId", }); + internal_static_jraft_GetLeaderResponse_descriptor = getDescriptor().getMessageTypes().get(10); + internal_static_jraft_GetLeaderResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetLeaderResponse_descriptor, new java.lang.String[] { "LeaderId", "ErrorResponse", }); + internal_static_jraft_GetPeersRequest_descriptor = getDescriptor().getMessageTypes().get(11); + internal_static_jraft_GetPeersRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetPeersRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "OnlyAlive", }); + internal_static_jraft_GetPeersResponse_descriptor = getDescriptor().getMessageTypes().get(12); + internal_static_jraft_GetPeersResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetPeersResponse_descriptor, new java.lang.String[] { "Peers", "Learners", + "ErrorResponse", }); + internal_static_jraft_AddLearnersRequest_descriptor = getDescriptor().getMessageTypes().get(13); + internal_static_jraft_AddLearnersRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AddLearnersRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "Learners", }); + internal_static_jraft_RemoveLearnersRequest_descriptor = getDescriptor().getMessageTypes().get(14); + internal_static_jraft_RemoveLearnersRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_RemoveLearnersRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "Learners", }); + internal_static_jraft_ResetLearnersRequest_descriptor = getDescriptor().getMessageTypes().get(15); + internal_static_jraft_ResetLearnersRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ResetLearnersRequest_descriptor, new java.lang.String[] { "GroupId", "LeaderId", + "Learners", }); + internal_static_jraft_LearnersOpResponse_descriptor = getDescriptor().getMessageTypes().get(16); + internal_static_jraft_LearnersOpResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_LearnersOpResponse_descriptor, new java.lang.String[] { "OldLearners", "NewLearners", + "ErrorResponse", }); + com.alipay.sofa.jraft.rpc.RpcRequests.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ClientService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ClientService.java new file mode 100644 index 0000000..3e3004c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ClientService.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Future; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.util.Endpoint; +import com.google.protobuf.Message; + +/** + * RPC client service + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 3:21:56 PM + */ +public interface ClientService extends Lifecycle { + + /** + * Connect to endpoint, returns true when success. + * + * @param endpoint server address + * @return true on connect success + */ + boolean connect(final Endpoint endpoint); + + /** + * Check connection for given address and async to create a new one if there is no connection. + * @param endpoint target address + * @param createIfAbsent create a new one if there is no connection + * @return true if there is a connection and the connection is active and writable. + */ + boolean checkConnection(final Endpoint endpoint, final boolean createIfAbsent); + + /** + * Disconnect from endpoint. + * + * @param endpoint server address + * @return true on disconnect success + */ + boolean disconnect(final Endpoint endpoint); + + /** + * Returns true when the endpoint's connection is active. + * + * @param endpoint server address + * @return true on connection is active + */ + boolean isConnected(final Endpoint endpoint); + + /** + * Send a requests and waits for response with callback, returns the request future. + * + * @param endpoint server address + * @param request request data + * @param done callback + * @param timeoutMs timeout millis + * @return a future with operation result + */ + Future invokeWithDone(final Endpoint endpoint, final Message request, + final RpcResponseClosure done, final int timeoutMs); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/Connection.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/Connection.java new file mode 100644 index 0000000..9bae02a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/Connection.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +/** + * + * RPC connection + * @author jiachun.fjc + */ +public interface Connection { + + /** + * Get the attribute that bound to the connection. + * + * @param key the attribute key + * @return the attribute value + */ + Object getAttribute(final String key); + + /** + * Set the attribute to the connection. + * + * @param key the attribute key + * @param value the attribute value + */ + void setAttribute(final String key, final Object value); + + /** + * Set the attribute to the connection if the key's item doesn't exist, otherwise returns the present item. + * + * @param key the attribute key + * @param value the attribute value + * @return the previous value associated with the specified key, or + * null if there was no mapping for the key. + */ + Object setAttributeIfAbsent(final String key, final Object value); + + /** + * Close the connection. + */ + void close(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeCallback.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeCallback.java new file mode 100644 index 0000000..88b0ef5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeCallback.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Executor; + +/** + * @author jiachun.fjc + */ +public interface InvokeCallback { + + void complete(final Object result, final Throwable err); + + default Executor executor() { + return null; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeContext.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeContext.java new file mode 100644 index 0000000..7d7e42e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/InvokeContext.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * RPC invoke context. + * + * @author jiachun.fjc + */ +public class InvokeContext { + + public final static String CRC_SWITCH = "invoke.crc.switch"; + + private final ConcurrentMap ctx = new ConcurrentHashMap<>(); + + public Object put(final String key, final Object value) { + return this.ctx.put(key, value); + } + + public Object putIfAbsent(final String key, final Object value) { + return this.ctx.putIfAbsent(key, value); + } + + @SuppressWarnings("unchecked") + public T get(final String key) { + return (T) this.ctx.get(key); + } + + @SuppressWarnings("unchecked") + public T getOrDefault(final String key, final T defaultValue) { + return (T) this.ctx.getOrDefault(key, defaultValue); + } + + public void clear() { + this.ctx.clear(); + } + + public Set> entrySet() { + return this.ctx.entrySet(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactory.java new file mode 100644 index 0000000..e081913 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactory.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.SerializationException; + +import com.alipay.sofa.jraft.error.MessageClassNotFoundException; +import com.alipay.sofa.jraft.storage.io.ProtoBufFile; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.DescriptorProtos.FileDescriptorProto; +import com.google.protobuf.DescriptorProtos.FileDescriptorSet; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FileDescriptor; +import com.google.protobuf.Message; + +import static java.lang.invoke.MethodType.methodType; + +/** + * Protobuf message factory. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-26 4:44:20 PM + */ +public class ProtobufMsgFactory { + + private static Map PARSE_METHODS_4PROTO = new HashMap<>(); + private static Map PARSE_METHODS_4J = new HashMap<>(); + private static Map DEFAULT_INSTANCE_METHODS_4J = new HashMap<>(); + + static { + try { + final FileDescriptorSet descriptorSet = FileDescriptorSet.parseFrom(ProtoBufFile.class + .getResourceAsStream("/raft.desc")); + final List resolveFDs = new ArrayList<>(); + final RaftRpcFactory rpcFactory = RpcFactoryHelper.rpcFactory(); + for (final FileDescriptorProto fdp : descriptorSet.getFileList()) { + + final FileDescriptor[] dependencies = new FileDescriptor[resolveFDs.size()]; + resolveFDs.toArray(dependencies); + + final FileDescriptor fd = FileDescriptor.buildFrom(fdp, dependencies); + resolveFDs.add(fd); + for (final Descriptor descriptor : fd.getMessageTypes()) { + + final String className = fdp.getOptions().getJavaPackage() + "." + + fdp.getOptions().getJavaOuterClassname() + "$" + descriptor.getName(); + final Class clazz = Class.forName(className); + final MethodHandle parseFromHandler = MethodHandles.lookup().findStatic(clazz, "parseFrom", + methodType(clazz, byte[].class)); + final MethodHandle getInstanceHandler = MethodHandles.lookup().findStatic(clazz, + "getDefaultInstance", methodType(clazz)); + PARSE_METHODS_4PROTO.put(descriptor.getFullName(), parseFromHandler); + PARSE_METHODS_4J.put(className, parseFromHandler); + DEFAULT_INSTANCE_METHODS_4J.put(className, getInstanceHandler); + rpcFactory.registerProtobufSerializer(className, getInstanceHandler.invoke()); + } + + } + } catch (final Throwable t) { + t.printStackTrace(); // NOPMD + } + } + + public static void load() { + if (PARSE_METHODS_4J.isEmpty() || PARSE_METHODS_4PROTO.isEmpty() || DEFAULT_INSTANCE_METHODS_4J.isEmpty()) { + throw new IllegalStateException("Parse protocol file failed."); + } + } + + @SuppressWarnings("unchecked") + public static T getDefaultInstance(final String className) { + final MethodHandle handle = DEFAULT_INSTANCE_METHODS_4J.get(className); + if (handle == null) { + throw new MessageClassNotFoundException(className + " not found"); + } + try { + return (T) handle.invoke(); + } catch (Throwable t) { + throw new SerializationException(t); + } + } + + @SuppressWarnings("unchecked") + public static T newMessageByJavaClassName(final String className, final byte[] bs) { + final MethodHandle handle = PARSE_METHODS_4J.get(className); + if (handle == null) { + throw new MessageClassNotFoundException(className + " not found"); + } + try { + return (T) handle.invoke(bs); + } catch (Throwable t) { + throw new SerializationException(t); + } + } + + @SuppressWarnings("unchecked") + public static T newMessageByProtoClassName(final String className, final byte[] bs) { + final MethodHandle handle = PARSE_METHODS_4PROTO.get(className); + if (handle == null) { + throw new MessageClassNotFoundException(className + " not found"); + } + try { + return (T) handle.invoke(bs); + } catch (Throwable t) { + throw new SerializationException(t); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufSerializer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufSerializer.java new file mode 100644 index 0000000..2d6e2bf --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/ProtobufSerializer.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.remoting.CustomSerializer; +import com.alipay.remoting.InvokeContext; +import com.alipay.remoting.exception.DeserializationException; +import com.alipay.remoting.exception.SerializationException; +import com.alipay.remoting.rpc.RequestCommand; +import com.alipay.remoting.rpc.ResponseCommand; +import com.alipay.remoting.rpc.protocol.RpcRequestCommand; +import com.alipay.remoting.rpc.protocol.RpcResponseCommand; +import com.google.protobuf.Message; + +/** + * RPC custom serializer based on protobuf + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-26 4:43:21 PM + */ +public class ProtobufSerializer implements CustomSerializer { + + public static final ProtobufSerializer INSTANCE = new ProtobufSerializer(); + + @Override + public boolean serializeHeader(T request, InvokeContext invokeContext) + throws SerializationException { + + final RpcRequestCommand cmd = (RpcRequestCommand) request; + final Message msg = (Message) cmd.getRequestObject(); + if (msg instanceof RpcRequests.AppendEntriesRequest) { + final RpcRequests.AppendEntriesRequest req = (RpcRequests.AppendEntriesRequest) msg; + final RpcRequests.AppendEntriesRequestHeader.Builder hb = RpcRequests.AppendEntriesRequestHeader + .newBuilder() // + .setGroupId(req.getGroupId()) // + .setPeerId(req.getPeerId()) // + .setServerId(req.getServerId()); + cmd.setHeader(hb.build().toByteArray()); + return true; + } + + return false; + } + + @Override + public boolean serializeHeader(T response) throws SerializationException { + return false; + } + + @Override + public boolean deserializeHeader(T request) throws DeserializationException { + final RpcRequestCommand cmd = (RpcRequestCommand) request; + final String className = cmd.getRequestClass(); + if (className.equals(RpcRequests.AppendEntriesRequest.class.getName())) { + final byte[] header = cmd.getHeader(); + cmd.setRequestHeader(ProtobufMsgFactory.newMessageByJavaClassName( + RpcRequests.AppendEntriesRequestHeader.class.getName(), header)); + return true; + } + return false; + } + + @Override + public boolean deserializeHeader(T response, InvokeContext invokeContext) + throws DeserializationException { + return false; + } + + @Override + public boolean serializeContent(T request, InvokeContext invokeContext) + throws SerializationException { + final RpcRequestCommand cmd = (RpcRequestCommand) request; + final Message msg = (Message) cmd.getRequestObject(); + cmd.setContent(msg.toByteArray()); + return true; + } + + @Override + public boolean serializeContent(T response) throws SerializationException { + final RpcResponseCommand cmd = (RpcResponseCommand) response; + final Message msg = (Message) cmd.getResponseObject(); + cmd.setContent(msg.toByteArray()); + return true; + } + + @Override + public boolean deserializeContent(T request) throws DeserializationException { + final RpcRequestCommand cmd = (RpcRequestCommand) request; + final String className = cmd.getRequestClass(); + + cmd.setRequestObject(ProtobufMsgFactory.newMessageByJavaClassName(className, request.getContent())); + return true; + } + + @Override + public boolean deserializeContent(T response, InvokeContext invokeContext) + throws DeserializationException { + final RpcResponseCommand cmd = (RpcResponseCommand) response; + final String className = cmd.getResponseClass(); + + cmd.setResponseObject(ProtobufMsgFactory.newMessageByJavaClassName(className, response.getContent())); + return true; + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftClientService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftClientService.java new file mode 100644 index 0000000..44e4b81 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftClientService.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Future; + +import com.alipay.sofa.jraft.util.Endpoint; +import com.google.protobuf.Message; + +/** + * Raft client RPC service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 5:59:52 PM + */ +public interface RaftClientService extends ClientService { + + /** + * Sends a pre-vote request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param done callback + * @return a future with result + */ + Future preVote(final Endpoint endpoint, final RpcRequests.RequestVoteRequest request, + final RpcResponseClosure done); + + /** + * Sends a request-vote request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param done callback + * @return a future with result + */ + Future requestVote(final Endpoint endpoint, final RpcRequests.RequestVoteRequest request, + final RpcResponseClosure done); + + /** + * Sends a append-entries request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param done callback + * @return a future with result + */ + Future appendEntries(final Endpoint endpoint, final RpcRequests.AppendEntriesRequest request, + final int timeoutMs, final RpcResponseClosure done); + + /** + * Sends a install-snapshot request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param done callback + * @return a future result + */ + Future installSnapshot(final Endpoint endpoint, final RpcRequests.InstallSnapshotRequest request, + final RpcResponseClosure done); + + /** + * Get a piece of file data by GetFileRequest, and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param timeoutMs timeout millis + * @param done callback + * @return a future result + */ + Future getFile(final Endpoint endpoint, final RpcRequests.GetFileRequest request, final int timeoutMs, + final RpcResponseClosure done); + + /** + * Send a timeout-now request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param timeoutMs timeout millis + * @param done callback + * @return a future result + */ + Future timeoutNow(final Endpoint endpoint, final RpcRequests.TimeoutNowRequest request, + final int timeoutMs, final RpcResponseClosure done); + + /** + * Send a read-index request and handle the response with done. + * + * @param endpoint destination address (ip, port) + * @param request request data + * @param timeoutMs timeout millis + * @param done callback + * @return a future result + */ + Future readIndex(final Endpoint endpoint, final RpcRequests.ReadIndexRequest request, final int timeoutMs, + final RpcResponseClosure done); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcFactory.java new file mode 100644 index 0000000..acc19c5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcFactory.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.util.Endpoint; + +/** + * Raft RPC service factory. + * + * @author jiachun.fjc + */ +public interface RaftRpcFactory { + + RpcResponseFactory DEFAULT = new RpcResponseFactory() {}; + + /** + * Register serializer with class name. + * + * @param className class name + * @param args extended parameters, different implementers may need different parameters, + * the order of parameters need a convention + */ + void registerProtobufSerializer(final String className, final Object... args); + + /** + * Creates a raft RPC client. + * + * @return a new rpc client instance + */ + default RpcClient createRpcClient() { + return createRpcClient(null); + } + + /** + * Creates a raft RPC client. + * + * @param helper config helper for rpc client impl + * @return a new rpc client instance + */ + RpcClient createRpcClient(final ConfigHelper helper); + + /** + * Creates a raft RPC server. + * + * @param endpoint server address to bind + * @return a new rpc server instance + */ + default RpcServer createRpcServer(final Endpoint endpoint) { + return createRpcServer(endpoint, null); + } + + /** + * Creates a raft RPC server. + * + * @param endpoint server address to bind + * @param helper config helper for rpc server impl + * @return a new rpc server instance + */ + RpcServer createRpcServer(final Endpoint endpoint, final ConfigHelper helper); + + default RpcResponseFactory getRpcResponseFactory() { + return DEFAULT; + } + + /** + * Whether to enable replicator pipeline. + * + * @return true if enable + */ + default boolean isReplicatorPipelineEnabled() { + return true; + } + + /** + * Ensure RPC framework supports pipeline. + */ + default void ensurePipeline() {} + + @SuppressWarnings("unused") + default ConfigHelper defaultJRaftClientConfigHelper(final RpcOptions opts) { + return null; + } + + @SuppressWarnings("unused") + default ConfigHelper defaultJRaftServerConfigHelper(final RpcOptions opts) { + return null; + } + + interface ConfigHelper { + + void config(final T instance); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcServerFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcServerFactory.java new file mode 100644 index 0000000..38dc6b6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftRpcServerFactory.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.impl.PingRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.AddLearnersRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.AddPeerRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.ChangePeersRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.GetLeaderRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.GetPeersRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.RemoveLearnersRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.RemovePeerRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.ResetLearnersRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.ResetPeerRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.SnapshotRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.cli.TransferLeaderRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.AppendEntriesRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.GetFileRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.InstallSnapshotRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.ReadIndexRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.RequestVoteRequestProcessor; +import com.alipay.sofa.jraft.rpc.impl.core.TimeoutNowRequestProcessor; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; + +/** + * Raft RPC server factory. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class RaftRpcServerFactory { + + static { + ProtobufMsgFactory.load(); + } + + /** + * Creates a raft RPC server with default request executors. + * + * @param endpoint server address to bind + * @return a rpc server instance + */ + public static RpcServer createRaftRpcServer(final Endpoint endpoint) { + return createRaftRpcServer(endpoint, null, null); + } + + /** + * Creates a raft RPC server with executors to handle requests. + * + * @param endpoint server address to bind + * @param raftExecutor executor to handle RAFT requests. + * @param cliExecutor executor to handle CLI service requests. + * @return a rpc server instance + */ + public static RpcServer createRaftRpcServer(final Endpoint endpoint, final Executor raftExecutor, + final Executor cliExecutor) { + final RpcServer rpcServer = RpcFactoryHelper.rpcFactory().createRpcServer(endpoint); + addRaftRequestProcessors(rpcServer, raftExecutor, cliExecutor); + return rpcServer; + } + + /** + * Adds RAFT and CLI service request processors with default executor. + * + * @param rpcServer rpc server instance + */ + public static void addRaftRequestProcessors(final RpcServer rpcServer) { + addRaftRequestProcessors(rpcServer, null, null); + } + + /** + * Adds RAFT and CLI service request processors. + * + * @param rpcServer rpc server instance + * @param raftExecutor executor to handle RAFT requests. + * @param cliExecutor executor to handle CLI service requests. + */ + public static void addRaftRequestProcessors(final RpcServer rpcServer, final Executor raftExecutor, + final Executor cliExecutor) { + // raft core processors + final AppendEntriesRequestProcessor appendEntriesRequestProcessor = new AppendEntriesRequestProcessor( + raftExecutor); + rpcServer.registerConnectionClosedEventListener(appendEntriesRequestProcessor); + rpcServer.registerProcessor(appendEntriesRequestProcessor); + rpcServer.registerProcessor(new GetFileRequestProcessor(raftExecutor)); + rpcServer.registerProcessor(new InstallSnapshotRequestProcessor(raftExecutor)); + rpcServer.registerProcessor(new RequestVoteRequestProcessor(raftExecutor)); + rpcServer.registerProcessor(new PingRequestProcessor()); + rpcServer.registerProcessor(new TimeoutNowRequestProcessor(raftExecutor)); + rpcServer.registerProcessor(new ReadIndexRequestProcessor(raftExecutor)); + // raft cli service + rpcServer.registerProcessor(new AddPeerRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new RemovePeerRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new ResetPeerRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new ChangePeersRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new GetLeaderRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new SnapshotRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new TransferLeaderRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new GetPeersRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new AddLearnersRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new RemoveLearnersRequestProcessor(cliExecutor)); + rpcServer.registerProcessor(new ResetLearnersRequestProcessor(cliExecutor)); + } + + /** + * Creates a raft RPC server and starts it. + * + * @param endpoint server address to bind + * @return a rpc server instance + */ + public static RpcServer createAndStartRaftRpcServer(final Endpoint endpoint) { + return createAndStartRaftRpcServer(endpoint, null, null); + } + + /** + * Creates a raft RPC server and starts it. + * + * @param endpoint server address to bind + * @param raftExecutor executor to handle RAFT requests. + * @param cliExecutor executor to handle CLI service requests. + * @return a rpc server instance + */ + public static RpcServer createAndStartRaftRpcServer(final Endpoint endpoint, final Executor raftExecutor, + final Executor cliExecutor) { + final RpcServer server = createRaftRpcServer(endpoint, raftExecutor, cliExecutor); + server.init(null); + return server; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftServerService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftServerService.java new file mode 100644 index 0000000..88563af --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RaftServerService.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest; +import com.google.protobuf.Message; + +/** + * Raft RPC service in server. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-03 4:25:09 PM + */ +public interface RaftServerService { + + /** + * Handle pre-vote request. + * + * @param request data of the pre vote + * @return the response message + */ + Message handlePreVoteRequest(RequestVoteRequest request); + + /** + * Handle request-vote request. + * + * @param request data of the vote + * @return the response message + */ + Message handleRequestVoteRequest(RequestVoteRequest request); + + /** + * Handle append-entries request, return response message or + * called done.run() with response. + * + * @param request data of the entries to append + * @param done callback + * @return the response message + */ + Message handleAppendEntriesRequest(AppendEntriesRequest request, RpcRequestClosure done); + + /** + * Handle install-snapshot request, return response message or + * called done.run() with response. + * + * @param request data of the install snapshot request + * @param done callback + * @return the response message + */ + Message handleInstallSnapshot(InstallSnapshotRequest request, RpcRequestClosure done); + + /** + * Handle time-out-now request, return response message or + * called done.run() with response. + * + * @param request data of the timeout now request + * @param done callback + * @return the response message + */ + Message handleTimeoutNowRequest(TimeoutNowRequest request, RpcRequestClosure done); + + /** + * Handle read-index request, call the RPC closure with response. + * + * @param request data of the readIndex read + * @param done callback + */ + void handleReadIndexRequest(ReadIndexRequest request, RpcResponseClosure done); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcClient.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcClient.java new file mode 100644 index 0000000..f6ae218 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcClient.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.util.Endpoint; + +/** + * + * @author jiachun.fjc + */ +public interface RpcClient extends Lifecycle { + + /** + * Check connection for given address. + * + * @param endpoint target address + * @return true if there is a connection and the connection is active and writable. + */ + boolean checkConnection(final Endpoint endpoint); + + /** + * Check connection for given address and async to create a new one if there is no connection. + * @param endpoint target address + * @param createIfAbsent create a new one if there is no connection + * @return true if there is a connection and the connection is active and writable. + */ + boolean checkConnection(final Endpoint endpoint, final boolean createIfAbsent); + + /** + * Close all connections of a address. + * + * @param endpoint target address + */ + void closeConnection(final Endpoint endpoint); + + /** + * Register a connect event listener for the replicator group. + * + * @param replicatorGroup replicator group + */ + void registerConnectEventListener(final ReplicatorGroup replicatorGroup); + + /** + * Synchronous invocation. + * + * @param endpoint target address + * @param request request object + * @param timeoutMs timeout millisecond + * @return invoke result + */ + default Object invokeSync(final Endpoint endpoint, final Object request, final long timeoutMs) + throws InterruptedException, RemotingException { + return invokeSync(endpoint, request, null, timeoutMs); + } + + /** + * Synchronous invocation using a invoke context. + * + * @param endpoint target address + * @param request request object + * @param ctx invoke context + * @param timeoutMs timeout millisecond + * @return invoke result + */ + Object invokeSync(final Endpoint endpoint, final Object request, final InvokeContext ctx, + final long timeoutMs) throws InterruptedException, RemotingException; + + /** + * Asynchronous invocation with a callback. + * + * @param endpoint target address + * @param request request object + * @param callback invoke callback + * @param timeoutMs timeout millisecond + */ + default void invokeAsync(final Endpoint endpoint, final Object request, final InvokeCallback callback, + final long timeoutMs) throws InterruptedException, RemotingException { + invokeAsync(endpoint, request, null, callback, timeoutMs); + } + + /** + * Asynchronous invocation with a callback. + * + * @param endpoint target address + * @param request request object + * @param ctx invoke context + * @param callback invoke callback + * @param timeoutMs timeout millisecond + */ + void invokeAsync(final Endpoint endpoint, final Object request, final InvokeContext ctx, final InvokeCallback callback, + final long timeoutMs) throws InterruptedException, RemotingException; +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcContext.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcContext.java new file mode 100644 index 0000000..e3d1e01 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcContext.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +/** + * @author jiachun.fjc + */ +public interface RpcContext { + + /** + * Send a response back. + * + * @param responseObj the response object + */ + void sendResponse(final Object responseObj); + + /** + * Get current connection. + * + * @return current connection + */ + Connection getConnection(); + + /** + * Get the remote address. + * + * @return remote address + */ + String getRemoteAddress(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcProcessor.java new file mode 100644 index 0000000..fdd8a5e --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcProcessor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Executor; + +/** + * Defined functions for process user defined request. + * + * @author jiachun.fjc + */ +public interface RpcProcessor { + + /** + * Async to handle request with {@link RpcContext}. + * + * @param rpcCtx the rpc context + * @param request the request + */ + void handleRequest(final RpcContext rpcCtx, final T request); + + /** + * The class name of user request. + * Use String type to avoid loading class. + * + * @return interested request's class name + */ + String interest(); + + /** + * Get user's executor. + * + * @return executor + */ + default Executor executor() { + return null; + } + + /** + * + * @return the executor selector + */ + default ExecutorSelector executorSelector() { + return null; + } + + /** + * Executor selector interface. + */ + interface ExecutorSelector { + + /** + * Select a executor. + * + * @param reqClass request class name + * @param reqHeader request header + * @return a executor + */ + Executor select(final String reqClass, final Object reqHeader); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestClosure.java new file mode 100644 index 0000000..f54cf4d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestClosure.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * RPC request Closure encapsulates the RPC contexts. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class RpcRequestClosure implements Closure { + + private static final Logger LOG = LoggerFactory + .getLogger(RpcRequestClosure.class); + + private static final AtomicIntegerFieldUpdater STATE_UPDATER = AtomicIntegerFieldUpdater + .newUpdater( + RpcRequestClosure.class, + "state"); + + private static final int PENDING = 0; + private static final int RESPOND = 1; + + private final RpcContext rpcCtx; + private final Message defaultResp; + + private volatile int state = PENDING; + + public RpcRequestClosure(RpcContext rpcCtx) { + this(rpcCtx, null); + } + + public RpcRequestClosure(RpcContext rpcCtx, Message defaultResp) { + super(); + this.rpcCtx = rpcCtx; + this.defaultResp = defaultResp; + } + + public RpcContext getRpcCtx() { + return rpcCtx; + } + + public void sendResponse(final Message msg) { + if (!STATE_UPDATER.compareAndSet(this, PENDING, RESPOND)) { + LOG.warn("A response: {} sent repeatedly!", msg); + return; + } + this.rpcCtx.sendResponse(msg); + } + + @Override + public void run(final Status status) { + sendResponse(RpcFactoryHelper.responseFactory().newResponse(this.defaultResp, status)); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestProcessor.java new file mode 100644 index 0000000..0fa74dc --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequestProcessor.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Executor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Abstract AsyncUserProcessor for RPC processors. + * + * @param Message + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public abstract class RpcRequestProcessor implements RpcProcessor { + + protected static final Logger LOG = LoggerFactory.getLogger(RpcRequestProcessor.class); + + private final Executor executor; + private final Message defaultResp; + + public abstract Message processRequest(final T request, final RpcRequestClosure done); + + public RpcRequestProcessor(Executor executor, Message defaultResp) { + super(); + this.executor = executor; + this.defaultResp = defaultResp; + } + + @Override + public void handleRequest(final RpcContext rpcCtx, final T request) { + try { + final Message msg = processRequest(request, new RpcRequestClosure(rpcCtx, this.defaultResp)); + if (msg != null) { + rpcCtx.sendResponse(msg); + } + } catch (final Throwable t) { + LOG.error("handleRequest {} failed", request, t); + rpcCtx.sendResponse(RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), -1, "handleRequest internal error")); + } + } + + @Override + public Executor executor() { + return this.executor; + } + + public Message defaultResp() { + return this.defaultResp; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java new file mode 100644 index 0000000..8d20d56 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcRequests.java @@ -0,0 +1,14824 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: rpc.proto + +package com.alipay.sofa.jraft.rpc; + +public final class RpcRequests { + private RpcRequests() { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry); + } + + public interface PingRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.PingRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 send_timestamp = 1; + */ + boolean hasSendTimestamp(); + + /** + * required int64 send_timestamp = 1; + */ + long getSendTimestamp(); + } + + /** + * Protobuf type {@code jraft.PingRequest} + */ + public static final class PingRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.PingRequest) + PingRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use PingRequest.newBuilder() to construct. + private PingRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private PingRequest() { + sendTimestamp_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private PingRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + sendTimestamp_ = input.readInt64(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_PingRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_PingRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.Builder.class); + } + + private int bitField0_; + public static final int SEND_TIMESTAMP_FIELD_NUMBER = 1; + private long sendTimestamp_; + + /** + * required int64 send_timestamp = 1; + */ + public boolean hasSendTimestamp() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 send_timestamp = 1; + */ + public long getSendTimestamp() { + return sendTimestamp_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasSendTimestamp()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, sendTimestamp_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, sendTimestamp_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest) obj; + + boolean result = true; + result = result && (hasSendTimestamp() == other.hasSendTimestamp()); + if (hasSendTimestamp()) { + result = result && (getSendTimestamp() == other.getSendTimestamp()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasSendTimestamp()) { + hash = (37 * hash) + SEND_TIMESTAMP_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getSendTimestamp()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.PingRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.PingRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_PingRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_PingRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + sendTimestamp_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_PingRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.sendTimestamp_ = sendTimestamp_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest.getDefaultInstance()) + return this; + if (other.hasSendTimestamp()) { + setSendTimestamp(other.getSendTimestamp()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasSendTimestamp()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long sendTimestamp_; + + /** + * required int64 send_timestamp = 1; + */ + public boolean hasSendTimestamp() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 send_timestamp = 1; + */ + public long getSendTimestamp() { + return sendTimestamp_; + } + + /** + * required int64 send_timestamp = 1; + */ + public Builder setSendTimestamp(long value) { + bitField0_ |= 0x00000001; + sendTimestamp_ = value; + onChanged(); + return this; + } + + /** + * required int64 send_timestamp = 1; + */ + public Builder clearSendTimestamp() { + bitField0_ = (bitField0_ & ~0x00000001); + sendTimestamp_ = 0L; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.PingRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.PingRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public PingRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new PingRequest(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ErrorResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ErrorResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int32 errorCode = 1; + */ + boolean hasErrorCode(); + + /** + * required int32 errorCode = 1; + */ + int getErrorCode(); + + /** + * optional string errorMsg = 2; + */ + boolean hasErrorMsg(); + + /** + * optional string errorMsg = 2; + */ + java.lang.String getErrorMsg(); + + /** + * optional string errorMsg = 2; + */ + com.google.protobuf.ByteString getErrorMsgBytes(); + } + + /** + * Protobuf type {@code jraft.ErrorResponse} + */ + public static final class ErrorResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ErrorResponse) + ErrorResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ErrorResponse.newBuilder() to construct. + private ErrorResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ErrorResponse() { + errorCode_ = 0; + errorMsg_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ErrorResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + errorCode_ = input.readInt32(); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + errorMsg_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ErrorResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ErrorResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder.class); + } + + private int bitField0_; + public static final int ERRORCODE_FIELD_NUMBER = 1; + private int errorCode_; + + /** + * required int32 errorCode = 1; + */ + public boolean hasErrorCode() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int32 errorCode = 1; + */ + public int getErrorCode() { + return errorCode_; + } + + public static final int ERRORMSG_FIELD_NUMBER = 2; + private volatile java.lang.Object errorMsg_; + + /** + * optional string errorMsg = 2; + */ + public boolean hasErrorMsg() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string errorMsg = 2; + */ + public java.lang.String getErrorMsg() { + java.lang.Object ref = errorMsg_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + errorMsg_ = s; + } + return s; + } + } + + /** + * optional string errorMsg = 2; + */ + public com.google.protobuf.ByteString getErrorMsgBytes() { + java.lang.Object ref = errorMsg_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + errorMsg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasErrorCode()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt32(1, errorCode_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, errorMsg_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt32Size(1, errorCode_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, errorMsg_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse) obj; + + boolean result = true; + result = result && (hasErrorCode() == other.hasErrorCode()); + if (hasErrorCode()) { + result = result && (getErrorCode() == other.getErrorCode()); + } + result = result && (hasErrorMsg() == other.hasErrorMsg()); + if (hasErrorMsg()) { + result = result && getErrorMsg().equals(other.getErrorMsg()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasErrorCode()) { + hash = (37 * hash) + ERRORCODE_FIELD_NUMBER; + hash = (53 * hash) + getErrorCode(); + } + if (hasErrorMsg()) { + hash = (37 * hash) + ERRORMSG_FIELD_NUMBER; + hash = (53 * hash) + getErrorMsg().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ErrorResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ErrorResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ErrorResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ErrorResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + errorCode_ = 0; + bitField0_ = (bitField0_ & ~0x00000001); + errorMsg_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ErrorResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.errorCode_ = errorCode_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.errorMsg_ = errorMsg_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) + return this; + if (other.hasErrorCode()) { + setErrorCode(other.getErrorCode()); + } + if (other.hasErrorMsg()) { + bitField0_ |= 0x00000002; + errorMsg_ = other.errorMsg_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasErrorCode()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private int errorCode_; + + /** + * required int32 errorCode = 1; + */ + public boolean hasErrorCode() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int32 errorCode = 1; + */ + public int getErrorCode() { + return errorCode_; + } + + /** + * required int32 errorCode = 1; + */ + public Builder setErrorCode(int value) { + bitField0_ |= 0x00000001; + errorCode_ = value; + onChanged(); + return this; + } + + /** + * required int32 errorCode = 1; + */ + public Builder clearErrorCode() { + bitField0_ = (bitField0_ & ~0x00000001); + errorCode_ = 0; + onChanged(); + return this; + } + + private java.lang.Object errorMsg_ = ""; + + /** + * optional string errorMsg = 2; + */ + public boolean hasErrorMsg() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * optional string errorMsg = 2; + */ + public java.lang.String getErrorMsg() { + java.lang.Object ref = errorMsg_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + errorMsg_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string errorMsg = 2; + */ + public com.google.protobuf.ByteString getErrorMsgBytes() { + java.lang.Object ref = errorMsg_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + errorMsg_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string errorMsg = 2; + */ + public Builder setErrorMsg(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + errorMsg_ = value; + onChanged(); + return this; + } + + /** + * optional string errorMsg = 2; + */ + public Builder clearErrorMsg() { + bitField0_ = (bitField0_ & ~0x00000002); + errorMsg_ = getDefaultInstance().getErrorMsg(); + onChanged(); + return this; + } + + /** + * optional string errorMsg = 2; + */ + public Builder setErrorMsgBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + errorMsg_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ErrorResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.ErrorResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ErrorResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ErrorResponse(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface InstallSnapshotRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.InstallSnapshotRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + + /** + * required int64 term = 4; + */ + boolean hasTerm(); + + /** + * required int64 term = 4; + */ + long getTerm(); + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + boolean hasMeta(); + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta(); + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder(); + + /** + * required string uri = 6; + */ + boolean hasUri(); + + /** + * required string uri = 6; + */ + java.lang.String getUri(); + + /** + * required string uri = 6; + */ + com.google.protobuf.ByteString getUriBytes(); + } + + /** + * Protobuf type {@code jraft.InstallSnapshotRequest} + */ + public static final class InstallSnapshotRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.InstallSnapshotRequest) + InstallSnapshotRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use InstallSnapshotRequest.newBuilder() to construct. + private InstallSnapshotRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private InstallSnapshotRequest() { + groupId_ = ""; + serverId_ = ""; + peerId_ = ""; + term_ = 0L; + uri_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private InstallSnapshotRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + case 32: { + bitField0_ |= 0x00000008; + term_ = input.readInt64(); + break; + } + case 42: { + com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder subBuilder = null; + if (((bitField0_ & 0x00000010) == 0x00000010)) { + subBuilder = meta_.toBuilder(); + } + meta_ = input.readMessage(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.PARSER, + extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(meta_); + meta_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000010; + break; + } + case 50: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000020; + uri_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TERM_FIELD_NUMBER = 4; + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + public static final int META_FIELD_NUMBER = 5; + private com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta meta_; + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta() { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() : meta_; + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder() { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() : meta_; + } + + public static final int URI_FIELD_NUMBER = 6; + private volatile java.lang.Object uri_; + + /** + * required string uri = 6; + */ + public boolean hasUri() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required string uri = 6; + */ + public java.lang.String getUri() { + java.lang.Object ref = uri_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + uri_ = s; + } + return s; + } + } + + /** + * required string uri = 6; + */ + public com.google.protobuf.ByteString getUriBytes() { + java.lang.Object ref = uri_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + uri_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasMeta()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasUri()) { + memoizedIsInitialized = 0; + return false; + } + if (!getMeta().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeMessage(5, getMeta()); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 6, uri_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(5, getMeta()); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(6, uri_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasMeta() == other.hasMeta()); + if (hasMeta()) { + result = result && getMeta().equals(other.getMeta()); + } + result = result && (hasUri() == other.hasUri()); + if (hasUri()) { + result = result && getUri().equals(other.getUri()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasMeta()) { + hash = (37 * hash) + META_FIELD_NUMBER; + hash = (53 * hash) + getMeta().hashCode(); + } + if (hasUri()) { + hash = (37 * hash) + URI_FIELD_NUMBER; + hash = (53 * hash) + getUri().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.InstallSnapshotRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.InstallSnapshotRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getMetaFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + if (metaBuilder_ == null) { + meta_ = null; + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000010); + uri_ = ""; + bitField0_ = (bitField0_ & ~0x00000020); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + if (metaBuilder_ == null) { + result.meta_ = meta_; + } else { + result.meta_ = metaBuilder_.build(); + } + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.uri_ = uri_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasMeta()) { + mergeMeta(other.getMeta()); + } + if (other.hasUri()) { + bitField0_ |= 0x00000020; + uri_ = other.uri_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + if (!hasTerm()) { + return false; + } + if (!hasMeta()) { + return false; + } + if (!hasUri()) { + return false; + } + if (!getMeta().isInitialized()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 4; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000008; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 4; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000008); + term_ = 0L; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta meta_ = null; + private com.google.protobuf.SingleFieldBuilderV3 metaBuilder_; + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public boolean hasMeta() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta getMeta() { + if (metaBuilder_ == null) { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() + : meta_; + } else { + return metaBuilder_.getMessage(); + } + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta value) { + if (metaBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + meta_ = value; + onChanged(); + } else { + metaBuilder_.setMessage(value); + } + bitField0_ |= 0x00000010; + return this; + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public Builder setMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder builderForValue) { + if (metaBuilder_ == null) { + meta_ = builderForValue.build(); + onChanged(); + } else { + metaBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000010; + return this; + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public Builder mergeMeta(com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta value) { + if (metaBuilder_ == null) { + if (((bitField0_ & 0x00000010) == 0x00000010) && meta_ != null + && meta_ != com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance()) { + meta_ = com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.newBuilder(meta_).mergeFrom(value) + .buildPartial(); + } else { + meta_ = value; + } + onChanged(); + } else { + metaBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000010; + return this; + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public Builder clearMeta() { + if (metaBuilder_ == null) { + meta_ = null; + onChanged(); + } else { + metaBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.Builder getMetaBuilder() { + bitField0_ |= 0x00000010; + onChanged(); + return getMetaFieldBuilder().getBuilder(); + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMetaOrBuilder getMetaOrBuilder() { + if (metaBuilder_ != null) { + return metaBuilder_.getMessageOrBuilder(); + } else { + return meta_ == null ? com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta.getDefaultInstance() + : meta_; + } + } + + /** + * required .jraft.SnapshotMeta meta = 5; + */ + private com.google.protobuf.SingleFieldBuilderV3 getMetaFieldBuilder() { + if (metaBuilder_ == null) { + metaBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getMeta(), getParentForChildren(), isClean()); + meta_ = null; + } + return metaBuilder_; + } + + private java.lang.Object uri_ = ""; + + /** + * required string uri = 6; + */ + public boolean hasUri() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required string uri = 6; + */ + public java.lang.String getUri() { + java.lang.Object ref = uri_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + uri_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string uri = 6; + */ + public com.google.protobuf.ByteString getUriBytes() { + java.lang.Object ref = uri_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + uri_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string uri = 6; + */ + public Builder setUri(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + uri_ = value; + onChanged(); + return this; + } + + /** + * required string uri = 6; + */ + public Builder clearUri() { + bitField0_ = (bitField0_ & ~0x00000020); + uri_ = getDefaultInstance().getUri(); + onChanged(); + return this; + } + + /** + * required string uri = 6; + */ + public Builder setUriBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000020; + uri_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.InstallSnapshotRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.InstallSnapshotRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public InstallSnapshotRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new InstallSnapshotRequest( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface InstallSnapshotResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.InstallSnapshotResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required bool success = 2; + */ + boolean hasSuccess(); + + /** + * required bool success = 2; + */ + boolean getSuccess(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.InstallSnapshotResponse} + */ + public static final class InstallSnapshotResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.InstallSnapshotResponse) + InstallSnapshotResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use InstallSnapshotResponse.newBuilder() to construct. + private InstallSnapshotResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private InstallSnapshotResponse() { + term_ = 0L; + success_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private InstallSnapshotResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + success_ = input.readBool(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000004; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int SUCCESS_FIELD_NUMBER = 2; + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasSuccess()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasSuccess() == other.hasSuccess()); + if (hasSuccess()) { + result = result && (getSuccess() == other.getSuccess()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasSuccess()) { + hash = (37 * hash) + SUCCESS_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getSuccess()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.InstallSnapshotResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.InstallSnapshotResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + success_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_InstallSnapshotResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.success_ = success_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasSuccess()) { + setSuccess(other.getSuccess()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasSuccess()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + /** + * required bool success = 2; + */ + public Builder setSuccess(boolean value) { + bitField0_ |= 0x00000002; + success_ = value; + onChanged(); + return this; + } + + /** + * required bool success = 2; + */ + public Builder clearSuccess() { + bitField0_ = (bitField0_ & ~0x00000002); + success_ = false; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.InstallSnapshotResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.InstallSnapshotResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public InstallSnapshotResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new InstallSnapshotResponse( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface TimeoutNowRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.TimeoutNowRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + + /** + * required int64 term = 4; + */ + boolean hasTerm(); + + /** + * required int64 term = 4; + */ + long getTerm(); + } + + /** + * Protobuf type {@code jraft.TimeoutNowRequest} + */ + public static final class TimeoutNowRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.TimeoutNowRequest) + TimeoutNowRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use TimeoutNowRequest.newBuilder() to construct. + private TimeoutNowRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private TimeoutNowRequest() { + groupId_ = ""; + serverId_ = ""; + peerId_ = ""; + term_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private TimeoutNowRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + case 32: { + bitField0_ |= 0x00000008; + term_ = input.readInt64(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TERM_FIELD_NUMBER = 4; + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, term_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, term_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.TimeoutNowRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.TimeoutNowRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.term_ = term_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + if (!hasTerm()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 4; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000008; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 4; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000008); + term_ = 0L; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.TimeoutNowRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.TimeoutNowRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public TimeoutNowRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new TimeoutNowRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface TimeoutNowResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.TimeoutNowResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required bool success = 2; + */ + boolean hasSuccess(); + + /** + * required bool success = 2; + */ + boolean getSuccess(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.TimeoutNowResponse} + */ + public static final class TimeoutNowResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.TimeoutNowResponse) + TimeoutNowResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use TimeoutNowResponse.newBuilder() to construct. + private TimeoutNowResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private TimeoutNowResponse() { + term_ = 0L; + success_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private TimeoutNowResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + success_ = input.readBool(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000004; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int SUCCESS_FIELD_NUMBER = 2; + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasSuccess()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasSuccess() == other.hasSuccess()); + if (hasSuccess()) { + result = result && (getSuccess() == other.getSuccess()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasSuccess()) { + hash = (37 * hash) + SUCCESS_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getSuccess()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.TimeoutNowResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.TimeoutNowResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + success_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_TimeoutNowResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.success_ = success_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasSuccess()) { + setSuccess(other.getSuccess()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasSuccess()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + /** + * required bool success = 2; + */ + public Builder setSuccess(boolean value) { + bitField0_ |= 0x00000002; + success_ = value; + onChanged(); + return this; + } + + /** + * required bool success = 2; + */ + public Builder clearSuccess() { + bitField0_ = (bitField0_ & ~0x00000002); + success_ = false; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.TimeoutNowResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.TimeoutNowResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public TimeoutNowResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new TimeoutNowResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface RequestVoteRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.RequestVoteRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + + /** + * required int64 term = 4; + */ + boolean hasTerm(); + + /** + * required int64 term = 4; + */ + long getTerm(); + + /** + * required int64 last_log_term = 5; + */ + boolean hasLastLogTerm(); + + /** + * required int64 last_log_term = 5; + */ + long getLastLogTerm(); + + /** + * required int64 last_log_index = 6; + */ + boolean hasLastLogIndex(); + + /** + * required int64 last_log_index = 6; + */ + long getLastLogIndex(); + + /** + * required bool pre_vote = 7; + */ + boolean hasPreVote(); + + /** + * required bool pre_vote = 7; + */ + boolean getPreVote(); + } + + /** + * Protobuf type {@code jraft.RequestVoteRequest} + */ + public static final class RequestVoteRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.RequestVoteRequest) + RequestVoteRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use RequestVoteRequest.newBuilder() to construct. + private RequestVoteRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private RequestVoteRequest() { + groupId_ = ""; + serverId_ = ""; + peerId_ = ""; + term_ = 0L; + lastLogTerm_ = 0L; + lastLogIndex_ = 0L; + preVote_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private RequestVoteRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + case 32: { + bitField0_ |= 0x00000008; + term_ = input.readInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + lastLogTerm_ = input.readInt64(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + lastLogIndex_ = input.readInt64(); + break; + } + case 56: { + bitField0_ |= 0x00000040; + preVote_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TERM_FIELD_NUMBER = 4; + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + public static final int LAST_LOG_TERM_FIELD_NUMBER = 5; + private long lastLogTerm_; + + /** + * required int64 last_log_term = 5; + */ + public boolean hasLastLogTerm() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required int64 last_log_term = 5; + */ + public long getLastLogTerm() { + return lastLogTerm_; + } + + public static final int LAST_LOG_INDEX_FIELD_NUMBER = 6; + private long lastLogIndex_; + + /** + * required int64 last_log_index = 6; + */ + public boolean hasLastLogIndex() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required int64 last_log_index = 6; + */ + public long getLastLogIndex() { + return lastLogIndex_; + } + + public static final int PRE_VOTE_FIELD_NUMBER = 7; + private boolean preVote_; + + /** + * required bool pre_vote = 7; + */ + public boolean hasPreVote() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + + /** + * required bool pre_vote = 7; + */ + public boolean getPreVote() { + return preVote_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLastLogTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasLastLogIndex()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPreVote()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(5, lastLogTerm_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt64(6, lastLogIndex_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeBool(7, preVote_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(5, lastLogTerm_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(6, lastLogIndex_); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(7, preVote_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasLastLogTerm() == other.hasLastLogTerm()); + if (hasLastLogTerm()) { + result = result && (getLastLogTerm() == other.getLastLogTerm()); + } + result = result && (hasLastLogIndex() == other.hasLastLogIndex()); + if (hasLastLogIndex()) { + result = result && (getLastLogIndex() == other.getLastLogIndex()); + } + result = result && (hasPreVote() == other.hasPreVote()); + if (hasPreVote()) { + result = result && (getPreVote() == other.getPreVote()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasLastLogTerm()) { + hash = (37 * hash) + LAST_LOG_TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getLastLogTerm()); + } + if (hasLastLogIndex()) { + hash = (37 * hash) + LAST_LOG_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getLastLogIndex()); + } + if (hasPreVote()) { + hash = (37 * hash) + PRE_VOTE_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getPreVote()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.RequestVoteRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.RequestVoteRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + lastLogTerm_ = 0L; + bitField0_ = (bitField0_ & ~0x00000010); + lastLogIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000020); + preVote_ = false; + bitField0_ = (bitField0_ & ~0x00000040); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.lastLogTerm_ = lastLogTerm_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.lastLogIndex_ = lastLogIndex_; + if (((from_bitField0_ & 0x00000040) == 0x00000040)) { + to_bitField0_ |= 0x00000040; + } + result.preVote_ = preVote_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasLastLogTerm()) { + setLastLogTerm(other.getLastLogTerm()); + } + if (other.hasLastLogIndex()) { + setLastLogIndex(other.getLastLogIndex()); + } + if (other.hasPreVote()) { + setPreVote(other.getPreVote()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + if (!hasTerm()) { + return false; + } + if (!hasLastLogTerm()) { + return false; + } + if (!hasLastLogIndex()) { + return false; + } + if (!hasPreVote()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 4; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000008; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 4; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000008); + term_ = 0L; + onChanged(); + return this; + } + + private long lastLogTerm_; + + /** + * required int64 last_log_term = 5; + */ + public boolean hasLastLogTerm() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required int64 last_log_term = 5; + */ + public long getLastLogTerm() { + return lastLogTerm_; + } + + /** + * required int64 last_log_term = 5; + */ + public Builder setLastLogTerm(long value) { + bitField0_ |= 0x00000010; + lastLogTerm_ = value; + onChanged(); + return this; + } + + /** + * required int64 last_log_term = 5; + */ + public Builder clearLastLogTerm() { + bitField0_ = (bitField0_ & ~0x00000010); + lastLogTerm_ = 0L; + onChanged(); + return this; + } + + private long lastLogIndex_; + + /** + * required int64 last_log_index = 6; + */ + public boolean hasLastLogIndex() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required int64 last_log_index = 6; + */ + public long getLastLogIndex() { + return lastLogIndex_; + } + + /** + * required int64 last_log_index = 6; + */ + public Builder setLastLogIndex(long value) { + bitField0_ |= 0x00000020; + lastLogIndex_ = value; + onChanged(); + return this; + } + + /** + * required int64 last_log_index = 6; + */ + public Builder clearLastLogIndex() { + bitField0_ = (bitField0_ & ~0x00000020); + lastLogIndex_ = 0L; + onChanged(); + return this; + } + + private boolean preVote_; + + /** + * required bool pre_vote = 7; + */ + public boolean hasPreVote() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + + /** + * required bool pre_vote = 7; + */ + public boolean getPreVote() { + return preVote_; + } + + /** + * required bool pre_vote = 7; + */ + public Builder setPreVote(boolean value) { + bitField0_ |= 0x00000040; + preVote_ = value; + onChanged(); + return this; + } + + /** + * required bool pre_vote = 7; + */ + public Builder clearPreVote() { + bitField0_ = (bitField0_ & ~0x00000040); + preVote_ = false; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.RequestVoteRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.RequestVoteRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public RequestVoteRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RequestVoteRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface RequestVoteResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.RequestVoteResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required bool granted = 2; + */ + boolean hasGranted(); + + /** + * required bool granted = 2; + */ + boolean getGranted(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.RequestVoteResponse} + */ + public static final class RequestVoteResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.RequestVoteResponse) + RequestVoteResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use RequestVoteResponse.newBuilder() to construct. + private RequestVoteResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private RequestVoteResponse() { + term_ = 0L; + granted_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private RequestVoteResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + granted_ = input.readBool(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000004; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int GRANTED_FIELD_NUMBER = 2; + private boolean granted_; + + /** + * required bool granted = 2; + */ + public boolean hasGranted() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool granted = 2; + */ + public boolean getGranted() { + return granted_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasGranted()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, granted_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, granted_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasGranted() == other.hasGranted()); + if (hasGranted()) { + result = result && (getGranted() == other.getGranted()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasGranted()) { + hash = (37 * hash) + GRANTED_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getGranted()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.RequestVoteResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.RequestVoteResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + granted_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_RequestVoteResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.granted_ = granted_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasGranted()) { + setGranted(other.getGranted()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasGranted()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private boolean granted_; + + /** + * required bool granted = 2; + */ + public boolean hasGranted() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool granted = 2; + */ + public boolean getGranted() { + return granted_; + } + + /** + * required bool granted = 2; + */ + public Builder setGranted(boolean value) { + bitField0_ |= 0x00000002; + granted_ = value; + onChanged(); + return this; + } + + /** + * required bool granted = 2; + */ + public Builder clearGranted() { + bitField0_ = (bitField0_ & ~0x00000002); + granted_ = false; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.RequestVoteResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.RequestVoteResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public RequestVoteResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RequestVoteResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface AppendEntriesRequestHeaderOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AppendEntriesRequestHeader) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.AppendEntriesRequestHeader} + */ + public static final class AppendEntriesRequestHeader extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AppendEntriesRequestHeader) + AppendEntriesRequestHeaderOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AppendEntriesRequestHeader.newBuilder() to construct. + private AppendEntriesRequestHeader(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AppendEntriesRequestHeader() { + groupId_ = ""; + serverId_ = ""; + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AppendEntriesRequestHeader(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequestHeader_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequestHeader_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader other = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AppendEntriesRequestHeader} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AppendEntriesRequestHeader) + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeaderOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequestHeader_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequestHeader_fieldAccessorTable + .ensureFieldAccessorsInitialized( + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequestHeader_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader build() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader result = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AppendEntriesRequestHeader) + } + + // @@protoc_insertion_point(class_scope:jraft.AppendEntriesRequestHeader) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AppendEntriesRequestHeader parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AppendEntriesRequestHeader( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface AppendEntriesRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AppendEntriesRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * required string peer_id = 3; + */ + boolean hasPeerId(); + + /** + * required string peer_id = 3; + */ + java.lang.String getPeerId(); + + /** + * required string peer_id = 3; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + + /** + * required int64 term = 4; + */ + boolean hasTerm(); + + /** + * required int64 term = 4; + */ + long getTerm(); + + /** + * required int64 prev_log_term = 5; + */ + boolean hasPrevLogTerm(); + + /** + * required int64 prev_log_term = 5; + */ + long getPrevLogTerm(); + + /** + * required int64 prev_log_index = 6; + */ + boolean hasPrevLogIndex(); + + /** + * required int64 prev_log_index = 6; + */ + long getPrevLogIndex(); + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + java.util.List getEntriesList(); + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getEntries(int index); + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + int getEntriesCount(); + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + java.util.List getEntriesOrBuilderList(); + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + com.alipay.sofa.jraft.entity.RaftOutter.EntryMetaOrBuilder getEntriesOrBuilder(int index); + + /** + * required int64 committed_index = 8; + */ + boolean hasCommittedIndex(); + + /** + * required int64 committed_index = 8; + */ + long getCommittedIndex(); + + /** + * optional bytes data = 9; + */ + boolean hasData(); + + /** + * optional bytes data = 9; + */ + com.google.protobuf.ByteString getData(); + } + + /** + * Protobuf type {@code jraft.AppendEntriesRequest} + */ + public static final class AppendEntriesRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AppendEntriesRequest) + AppendEntriesRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AppendEntriesRequest.newBuilder() to construct. + private AppendEntriesRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AppendEntriesRequest() { + groupId_ = ""; + serverId_ = ""; + peerId_ = ""; + term_ = 0L; + prevLogTerm_ = 0L; + prevLogIndex_ = 0L; + entries_ = java.util.Collections.emptyList(); + committedIndex_ = 0L; + data_ = com.google.protobuf.ByteString.EMPTY; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AppendEntriesRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + case 32: { + bitField0_ |= 0x00000008; + term_ = input.readInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + prevLogTerm_ = input.readInt64(); + break; + } + case 48: { + bitField0_ |= 0x00000020; + prevLogIndex_ = input.readInt64(); + break; + } + case 58: { + if (!((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + entries_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000040; + } + entries_.add(input.readMessage(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.PARSER, + extensionRegistry)); + break; + } + case 64: { + bitField0_ |= 0x00000040; + committedIndex_ = input.readInt64(); + break; + } + case 74: { + bitField0_ |= 0x00000080; + data_ = input.readBytes(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000040) == 0x00000040)) { + entries_ = java.util.Collections.unmodifiableList(entries_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int PEER_ID_FIELD_NUMBER = 3; + private volatile java.lang.Object peerId_; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int TERM_FIELD_NUMBER = 4; + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + public static final int PREV_LOG_TERM_FIELD_NUMBER = 5; + private long prevLogTerm_; + + /** + * required int64 prev_log_term = 5; + */ + public boolean hasPrevLogTerm() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required int64 prev_log_term = 5; + */ + public long getPrevLogTerm() { + return prevLogTerm_; + } + + public static final int PREV_LOG_INDEX_FIELD_NUMBER = 6; + private long prevLogIndex_; + + /** + * required int64 prev_log_index = 6; + */ + public boolean hasPrevLogIndex() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required int64 prev_log_index = 6; + */ + public long getPrevLogIndex() { + return prevLogIndex_; + } + + public static final int ENTRIES_FIELD_NUMBER = 7; + private java.util.List entries_; + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public java.util.List getEntriesList() { + return entries_; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public java.util.List getEntriesOrBuilderList() { + return entries_; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public int getEntriesCount() { + return entries_.size(); + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getEntries(int index) { + return entries_.get(index); + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMetaOrBuilder getEntriesOrBuilder(int index) { + return entries_.get(index); + } + + public static final int COMMITTED_INDEX_FIELD_NUMBER = 8; + private long committedIndex_; + + /** + * required int64 committed_index = 8; + */ + public boolean hasCommittedIndex() { + return ((bitField0_ & 0x00000040) == 0x00000040); + } + + /** + * required int64 committed_index = 8; + */ + public long getCommittedIndex() { + return committedIndex_; + } + + public static final int DATA_FIELD_NUMBER = 9; + private com.google.protobuf.ByteString data_; + + /** + * optional bytes data = 9; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + + /** + * optional bytes data = 9; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPeerId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPrevLogTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasPrevLogIndex()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasCommittedIndex()) { + memoizedIsInitialized = 0; + return false; + } + for (int i = 0; i < getEntriesCount(); i++) { + if (!getEntries(i).isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeInt64(5, prevLogTerm_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + output.writeInt64(6, prevLogIndex_); + } + for (int i = 0; i < entries_.size(); i++) { + output.writeMessage(7, entries_.get(i)); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + output.writeInt64(8, committedIndex_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + output.writeBytes(9, data_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, peerId_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, term_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(5, prevLogTerm_); + } + if (((bitField0_ & 0x00000020) == 0x00000020)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(6, prevLogIndex_); + } + for (int i = 0; i < entries_.size(); i++) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(7, entries_.get(i)); + } + if (((bitField0_ & 0x00000040) == 0x00000040)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(8, committedIndex_); + } + if (((bitField0_ & 0x00000080) == 0x00000080)) { + size += com.google.protobuf.CodedOutputStream.computeBytesSize(9, data_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasPrevLogTerm() == other.hasPrevLogTerm()); + if (hasPrevLogTerm()) { + result = result && (getPrevLogTerm() == other.getPrevLogTerm()); + } + result = result && (hasPrevLogIndex() == other.hasPrevLogIndex()); + if (hasPrevLogIndex()) { + result = result && (getPrevLogIndex() == other.getPrevLogIndex()); + } + result = result && getEntriesList().equals(other.getEntriesList()); + result = result && (hasCommittedIndex() == other.hasCommittedIndex()); + if (hasCommittedIndex()) { + result = result && (getCommittedIndex() == other.getCommittedIndex()); + } + result = result && (hasData() == other.hasData()); + if (hasData()) { + result = result && getData().equals(other.getData()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasPrevLogTerm()) { + hash = (37 * hash) + PREV_LOG_TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getPrevLogTerm()); + } + if (hasPrevLogIndex()) { + hash = (37 * hash) + PREV_LOG_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getPrevLogIndex()); + } + if (getEntriesCount() > 0) { + hash = (37 * hash) + ENTRIES_FIELD_NUMBER; + hash = (53 * hash) + getEntriesList().hashCode(); + } + if (hasCommittedIndex()) { + hash = (37 * hash) + COMMITTED_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getCommittedIndex()); + } + if (hasData()) { + hash = (37 * hash) + DATA_FIELD_NUMBER; + hash = (53 * hash) + getData().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AppendEntriesRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AppendEntriesRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getEntriesFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000004); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + prevLogTerm_ = 0L; + bitField0_ = (bitField0_ & ~0x00000010); + prevLogIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000020); + if (entriesBuilder_ == null) { + entries_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + } else { + entriesBuilder_.clear(); + } + committedIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000080); + data_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000100); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.prevLogTerm_ = prevLogTerm_; + if (((from_bitField0_ & 0x00000020) == 0x00000020)) { + to_bitField0_ |= 0x00000020; + } + result.prevLogIndex_ = prevLogIndex_; + if (entriesBuilder_ == null) { + if (((bitField0_ & 0x00000040) == 0x00000040)) { + entries_ = java.util.Collections.unmodifiableList(entries_); + bitField0_ = (bitField0_ & ~0x00000040); + } + result.entries_ = entries_; + } else { + result.entries_ = entriesBuilder_.build(); + } + if (((from_bitField0_ & 0x00000080) == 0x00000080)) { + to_bitField0_ |= 0x00000040; + } + result.committedIndex_ = committedIndex_; + if (((from_bitField0_ & 0x00000100) == 0x00000100)) { + to_bitField0_ |= 0x00000080; + } + result.data_ = data_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000004; + peerId_ = other.peerId_; + onChanged(); + } + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasPrevLogTerm()) { + setPrevLogTerm(other.getPrevLogTerm()); + } + if (other.hasPrevLogIndex()) { + setPrevLogIndex(other.getPrevLogIndex()); + } + if (entriesBuilder_ == null) { + if (!other.entries_.isEmpty()) { + if (entries_.isEmpty()) { + entries_ = other.entries_; + bitField0_ = (bitField0_ & ~0x00000040); + } else { + ensureEntriesIsMutable(); + entries_.addAll(other.entries_); + } + onChanged(); + } + } else { + if (!other.entries_.isEmpty()) { + if (entriesBuilder_.isEmpty()) { + entriesBuilder_.dispose(); + entriesBuilder_ = null; + entries_ = other.entries_; + bitField0_ = (bitField0_ & ~0x00000040); + entriesBuilder_ = com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders ? getEntriesFieldBuilder() + : null; + } else { + entriesBuilder_.addAllMessages(other.entries_); + } + } + } + if (other.hasCommittedIndex()) { + setCommittedIndex(other.getCommittedIndex()); + } + if (other.hasData()) { + setData(other.getData()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + if (!hasPeerId()) { + return false; + } + if (!hasTerm()) { + return false; + } + if (!hasPrevLogTerm()) { + return false; + } + if (!hasPrevLogIndex()) { + return false; + } + if (!hasCommittedIndex()) { + return false; + } + for (int i = 0; i < getEntriesCount(); i++) { + if (!getEntries(i).isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * required string peer_id = 3; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required string peer_id = 3; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string peer_id = 3; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * required string peer_id = 3; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000004; + peerId_ = value; + onChanged(); + return this; + } + + private long term_; + + /** + * required int64 term = 4; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 term = 4; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 4; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000008; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 4; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000008); + term_ = 0L; + onChanged(); + return this; + } + + private long prevLogTerm_; + + /** + * required int64 prev_log_term = 5; + */ + public boolean hasPrevLogTerm() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * required int64 prev_log_term = 5; + */ + public long getPrevLogTerm() { + return prevLogTerm_; + } + + /** + * required int64 prev_log_term = 5; + */ + public Builder setPrevLogTerm(long value) { + bitField0_ |= 0x00000010; + prevLogTerm_ = value; + onChanged(); + return this; + } + + /** + * required int64 prev_log_term = 5; + */ + public Builder clearPrevLogTerm() { + bitField0_ = (bitField0_ & ~0x00000010); + prevLogTerm_ = 0L; + onChanged(); + return this; + } + + private long prevLogIndex_; + + /** + * required int64 prev_log_index = 6; + */ + public boolean hasPrevLogIndex() { + return ((bitField0_ & 0x00000020) == 0x00000020); + } + + /** + * required int64 prev_log_index = 6; + */ + public long getPrevLogIndex() { + return prevLogIndex_; + } + + /** + * required int64 prev_log_index = 6; + */ + public Builder setPrevLogIndex(long value) { + bitField0_ |= 0x00000020; + prevLogIndex_ = value; + onChanged(); + return this; + } + + /** + * required int64 prev_log_index = 6; + */ + public Builder clearPrevLogIndex() { + bitField0_ = (bitField0_ & ~0x00000020); + prevLogIndex_ = 0L; + onChanged(); + return this; + } + + private java.util.List entries_ = java.util.Collections + .emptyList(); + + private void ensureEntriesIsMutable() { + if (!((bitField0_ & 0x00000040) == 0x00000040)) { + entries_ = new java.util.ArrayList(entries_); + bitField0_ |= 0x00000040; + } + } + + private com.google.protobuf.RepeatedFieldBuilderV3 entriesBuilder_; + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public java.util.List getEntriesList() { + if (entriesBuilder_ == null) { + return java.util.Collections.unmodifiableList(entries_); + } else { + return entriesBuilder_.getMessageList(); + } + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public int getEntriesCount() { + if (entriesBuilder_ == null) { + return entries_.size(); + } else { + return entriesBuilder_.getCount(); + } + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta getEntries(int index) { + if (entriesBuilder_ == null) { + return entries_.get(index); + } else { + return entriesBuilder_.getMessage(index); + } + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder setEntries(int index, com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta value) { + if (entriesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntriesIsMutable(); + entries_.set(index, value); + onChanged(); + } else { + entriesBuilder_.setMessage(index, value); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder setEntries(int index, + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder builderForValue) { + if (entriesBuilder_ == null) { + ensureEntriesIsMutable(); + entries_.set(index, builderForValue.build()); + onChanged(); + } else { + entriesBuilder_.setMessage(index, builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder addEntries(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta value) { + if (entriesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntriesIsMutable(); + entries_.add(value); + onChanged(); + } else { + entriesBuilder_.addMessage(value); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder addEntries(int index, com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta value) { + if (entriesBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntriesIsMutable(); + entries_.add(index, value); + onChanged(); + } else { + entriesBuilder_.addMessage(index, value); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder addEntries(com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder builderForValue) { + if (entriesBuilder_ == null) { + ensureEntriesIsMutable(); + entries_.add(builderForValue.build()); + onChanged(); + } else { + entriesBuilder_.addMessage(builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder addEntries(int index, + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder builderForValue) { + if (entriesBuilder_ == null) { + ensureEntriesIsMutable(); + entries_.add(index, builderForValue.build()); + onChanged(); + } else { + entriesBuilder_.addMessage(index, builderForValue.build()); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder addAllEntries(java.lang.Iterable values) { + if (entriesBuilder_ == null) { + ensureEntriesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, entries_); + onChanged(); + } else { + entriesBuilder_.addAllMessages(values); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder clearEntries() { + if (entriesBuilder_ == null) { + entries_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000040); + onChanged(); + } else { + entriesBuilder_.clear(); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public Builder removeEntries(int index) { + if (entriesBuilder_ == null) { + ensureEntriesIsMutable(); + entries_.remove(index); + onChanged(); + } else { + entriesBuilder_.remove(index); + } + return this; + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder getEntriesBuilder(int index) { + return getEntriesFieldBuilder().getBuilder(index); + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMetaOrBuilder getEntriesOrBuilder(int index) { + if (entriesBuilder_ == null) { + return entries_.get(index); + } else { + return entriesBuilder_.getMessageOrBuilder(index); + } + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public java.util.List getEntriesOrBuilderList() { + if (entriesBuilder_ != null) { + return entriesBuilder_.getMessageOrBuilderList(); + } else { + return java.util.Collections.unmodifiableList(entries_); + } + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder addEntriesBuilder() { + return getEntriesFieldBuilder().addBuilder( + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.getDefaultInstance()); + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.Builder addEntriesBuilder(int index) { + return getEntriesFieldBuilder().addBuilder(index, + com.alipay.sofa.jraft.entity.RaftOutter.EntryMeta.getDefaultInstance()); + } + + /** + * repeated .jraft.EntryMeta entries = 7; + */ + public java.util.List getEntriesBuilderList() { + return getEntriesFieldBuilder().getBuilderList(); + } + + private com.google.protobuf.RepeatedFieldBuilderV3 getEntriesFieldBuilder() { + if (entriesBuilder_ == null) { + entriesBuilder_ = new com.google.protobuf.RepeatedFieldBuilderV3( + entries_, ((bitField0_ & 0x00000040) == 0x00000040), getParentForChildren(), isClean()); + entries_ = null; + } + return entriesBuilder_; + } + + private long committedIndex_; + + /** + * required int64 committed_index = 8; + */ + public boolean hasCommittedIndex() { + return ((bitField0_ & 0x00000080) == 0x00000080); + } + + /** + * required int64 committed_index = 8; + */ + public long getCommittedIndex() { + return committedIndex_; + } + + /** + * required int64 committed_index = 8; + */ + public Builder setCommittedIndex(long value) { + bitField0_ |= 0x00000080; + committedIndex_ = value; + onChanged(); + return this; + } + + /** + * required int64 committed_index = 8; + */ + public Builder clearCommittedIndex() { + bitField0_ = (bitField0_ & ~0x00000080); + committedIndex_ = 0L; + onChanged(); + return this; + } + + private com.google.protobuf.ByteString data_ = com.google.protobuf.ByteString.EMPTY; + + /** + * optional bytes data = 9; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000100) == 0x00000100); + } + + /** + * optional bytes data = 9; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + /** + * optional bytes data = 9; + */ + public Builder setData(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000100; + data_ = value; + onChanged(); + return this; + } + + /** + * optional bytes data = 9; + */ + public Builder clearData() { + bitField0_ = (bitField0_ & ~0x00000100); + data_ = getDefaultInstance().getData(); + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AppendEntriesRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.AppendEntriesRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AppendEntriesRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AppendEntriesRequest( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface AppendEntriesResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.AppendEntriesResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 term = 1; + */ + boolean hasTerm(); + + /** + * required int64 term = 1; + */ + long getTerm(); + + /** + * required bool success = 2; + */ + boolean hasSuccess(); + + /** + * required bool success = 2; + */ + boolean getSuccess(); + + /** + * optional int64 last_log_index = 3; + */ + boolean hasLastLogIndex(); + + /** + * optional int64 last_log_index = 3; + */ + long getLastLogIndex(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.AppendEntriesResponse} + */ + public static final class AppendEntriesResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.AppendEntriesResponse) + AppendEntriesResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use AppendEntriesResponse.newBuilder() to construct. + private AppendEntriesResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private AppendEntriesResponse() { + term_ = 0L; + success_ = false; + lastLogIndex_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private AppendEntriesResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + term_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + success_ = input.readBool(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + lastLogIndex_ = input.readInt64(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000008) == 0x00000008)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000008; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.Builder.class); + } + + private int bitField0_; + public static final int TERM_FIELD_NUMBER = 1; + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + public static final int SUCCESS_FIELD_NUMBER = 2; + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + public static final int LAST_LOG_INDEX_FIELD_NUMBER = 3; + private long lastLogIndex_; + + /** + * optional int64 last_log_index = 3; + */ + public boolean hasLastLogIndex() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional int64 last_log_index = 3; + */ + public long getLastLogIndex() { + return lastLogIndex_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasTerm()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasSuccess()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, lastLogIndex_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, term_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(3, lastLogIndex_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse) obj; + + boolean result = true; + result = result && (hasTerm() == other.hasTerm()); + if (hasTerm()) { + result = result && (getTerm() == other.getTerm()); + } + result = result && (hasSuccess() == other.hasSuccess()); + if (hasSuccess()) { + result = result && (getSuccess() == other.getSuccess()); + } + result = result && (hasLastLogIndex() == other.hasLastLogIndex()); + if (hasLastLogIndex()) { + result = result && (getLastLogIndex() == other.getLastLogIndex()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasTerm()) { + hash = (37 * hash) + TERM_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getTerm()); + } + if (hasSuccess()) { + hash = (37 * hash) + SUCCESS_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getSuccess()); + } + if (hasLastLogIndex()) { + hash = (37 * hash) + LAST_LOG_INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getLastLogIndex()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.AppendEntriesResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.AppendEntriesResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + term_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + success_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + lastLogIndex_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_AppendEntriesResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.term_ = term_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.success_ = success_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.lastLogIndex_ = lastLogIndex_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse.getDefaultInstance()) + return this; + if (other.hasTerm()) { + setTerm(other.getTerm()); + } + if (other.hasSuccess()) { + setSuccess(other.getSuccess()); + } + if (other.hasLastLogIndex()) { + setLastLogIndex(other.getLastLogIndex()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasTerm()) { + return false; + } + if (!hasSuccess()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse) e + .getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long term_; + + /** + * required int64 term = 1; + */ + public boolean hasTerm() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 term = 1; + */ + public long getTerm() { + return term_; + } + + /** + * required int64 term = 1; + */ + public Builder setTerm(long value) { + bitField0_ |= 0x00000001; + term_ = value; + onChanged(); + return this; + } + + /** + * required int64 term = 1; + */ + public Builder clearTerm() { + bitField0_ = (bitField0_ & ~0x00000001); + term_ = 0L; + onChanged(); + return this; + } + + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + /** + * required bool success = 2; + */ + public Builder setSuccess(boolean value) { + bitField0_ |= 0x00000002; + success_ = value; + onChanged(); + return this; + } + + /** + * required bool success = 2; + */ + public Builder clearSuccess() { + bitField0_ = (bitField0_ & ~0x00000002); + success_ = false; + onChanged(); + return this; + } + + private long lastLogIndex_; + + /** + * optional int64 last_log_index = 3; + */ + public boolean hasLastLogIndex() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional int64 last_log_index = 3; + */ + public long getLastLogIndex() { + return lastLogIndex_; + } + + /** + * optional int64 last_log_index = 3; + */ + public Builder setLastLogIndex(long value) { + bitField0_ |= 0x00000004; + lastLogIndex_ = value; + onChanged(); + return this; + } + + /** + * optional int64 last_log_index = 3; + */ + public Builder clearLastLogIndex() { + bitField0_ = (bitField0_ & ~0x00000004); + lastLogIndex_ = 0L; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000008) == 0x00000008) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000008; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.AppendEntriesResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.AppendEntriesResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public AppendEntriesResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new AppendEntriesResponse( + input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetFileRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetFileRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 reader_id = 1; + */ + boolean hasReaderId(); + + /** + * required int64 reader_id = 1; + */ + long getReaderId(); + + /** + * required string filename = 2; + */ + boolean hasFilename(); + + /** + * required string filename = 2; + */ + java.lang.String getFilename(); + + /** + * required string filename = 2; + */ + com.google.protobuf.ByteString getFilenameBytes(); + + /** + * required int64 count = 3; + */ + boolean hasCount(); + + /** + * required int64 count = 3; + */ + long getCount(); + + /** + * required int64 offset = 4; + */ + boolean hasOffset(); + + /** + * required int64 offset = 4; + */ + long getOffset(); + + /** + * optional bool read_partly = 5; + */ + boolean hasReadPartly(); + + /** + * optional bool read_partly = 5; + */ + boolean getReadPartly(); + } + + /** + * Protobuf type {@code jraft.GetFileRequest} + */ + public static final class GetFileRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetFileRequest) + GetFileRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetFileRequest.newBuilder() to construct. + private GetFileRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetFileRequest() { + readerId_ = 0L; + filename_ = ""; + count_ = 0L; + offset_ = 0L; + readPartly_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetFileRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + readerId_ = input.readInt64(); + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + filename_ = bs; + break; + } + case 24: { + bitField0_ |= 0x00000004; + count_ = input.readInt64(); + break; + } + case 32: { + bitField0_ |= 0x00000008; + offset_ = input.readInt64(); + break; + } + case 40: { + bitField0_ |= 0x00000010; + readPartly_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.Builder.class); + } + + private int bitField0_; + public static final int READER_ID_FIELD_NUMBER = 1; + private long readerId_; + + /** + * required int64 reader_id = 1; + */ + public boolean hasReaderId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 reader_id = 1; + */ + public long getReaderId() { + return readerId_; + } + + public static final int FILENAME_FIELD_NUMBER = 2; + private volatile java.lang.Object filename_; + + /** + * required string filename = 2; + */ + public boolean hasFilename() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string filename = 2; + */ + public java.lang.String getFilename() { + java.lang.Object ref = filename_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + filename_ = s; + } + return s; + } + } + + /** + * required string filename = 2; + */ + public com.google.protobuf.ByteString getFilenameBytes() { + java.lang.Object ref = filename_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + filename_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int COUNT_FIELD_NUMBER = 3; + private long count_; + + /** + * required int64 count = 3; + */ + public boolean hasCount() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required int64 count = 3; + */ + public long getCount() { + return count_; + } + + public static final int OFFSET_FIELD_NUMBER = 4; + private long offset_; + + /** + * required int64 offset = 4; + */ + public boolean hasOffset() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 offset = 4; + */ + public long getOffset() { + return offset_; + } + + public static final int READ_PARTLY_FIELD_NUMBER = 5; + private boolean readPartly_; + + /** + * optional bool read_partly = 5; + */ + public boolean hasReadPartly() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * optional bool read_partly = 5; + */ + public boolean getReadPartly() { + return readPartly_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasReaderId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasFilename()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasCount()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasOffset()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, readerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, filename_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, count_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeInt64(4, offset_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + output.writeBool(5, readPartly_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, readerId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, filename_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(3, count_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(4, offset_); + } + if (((bitField0_ & 0x00000010) == 0x00000010)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(5, readPartly_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest) obj; + + boolean result = true; + result = result && (hasReaderId() == other.hasReaderId()); + if (hasReaderId()) { + result = result && (getReaderId() == other.getReaderId()); + } + result = result && (hasFilename() == other.hasFilename()); + if (hasFilename()) { + result = result && getFilename().equals(other.getFilename()); + } + result = result && (hasCount() == other.hasCount()); + if (hasCount()) { + result = result && (getCount() == other.getCount()); + } + result = result && (hasOffset() == other.hasOffset()); + if (hasOffset()) { + result = result && (getOffset() == other.getOffset()); + } + result = result && (hasReadPartly() == other.hasReadPartly()); + if (hasReadPartly()) { + result = result && (getReadPartly() == other.getReadPartly()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasReaderId()) { + hash = (37 * hash) + READER_ID_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getReaderId()); + } + if (hasFilename()) { + hash = (37 * hash) + FILENAME_FIELD_NUMBER; + hash = (53 * hash) + getFilename().hashCode(); + } + if (hasCount()) { + hash = (37 * hash) + COUNT_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getCount()); + } + if (hasOffset()) { + hash = (37 * hash) + OFFSET_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getOffset()); + } + if (hasReadPartly()) { + hash = (37 * hash) + READ_PARTLY_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getReadPartly()); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetFileRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetFileRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + readerId_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + filename_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + count_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + offset_ = 0L; + bitField0_ = (bitField0_ & ~0x00000008); + readPartly_ = false; + bitField0_ = (bitField0_ & ~0x00000010); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.readerId_ = readerId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.filename_ = filename_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.count_ = count_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + result.offset_ = offset_; + if (((from_bitField0_ & 0x00000010) == 0x00000010)) { + to_bitField0_ |= 0x00000010; + } + result.readPartly_ = readPartly_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest.getDefaultInstance()) + return this; + if (other.hasReaderId()) { + setReaderId(other.getReaderId()); + } + if (other.hasFilename()) { + bitField0_ |= 0x00000002; + filename_ = other.filename_; + onChanged(); + } + if (other.hasCount()) { + setCount(other.getCount()); + } + if (other.hasOffset()) { + setOffset(other.getOffset()); + } + if (other.hasReadPartly()) { + setReadPartly(other.getReadPartly()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasReaderId()) { + return false; + } + if (!hasFilename()) { + return false; + } + if (!hasCount()) { + return false; + } + if (!hasOffset()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long readerId_; + + /** + * required int64 reader_id = 1; + */ + public boolean hasReaderId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 reader_id = 1; + */ + public long getReaderId() { + return readerId_; + } + + /** + * required int64 reader_id = 1; + */ + public Builder setReaderId(long value) { + bitField0_ |= 0x00000001; + readerId_ = value; + onChanged(); + return this; + } + + /** + * required int64 reader_id = 1; + */ + public Builder clearReaderId() { + bitField0_ = (bitField0_ & ~0x00000001); + readerId_ = 0L; + onChanged(); + return this; + } + + private java.lang.Object filename_ = ""; + + /** + * required string filename = 2; + */ + public boolean hasFilename() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string filename = 2; + */ + public java.lang.String getFilename() { + java.lang.Object ref = filename_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + filename_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string filename = 2; + */ + public com.google.protobuf.ByteString getFilenameBytes() { + java.lang.Object ref = filename_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + filename_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string filename = 2; + */ + public Builder setFilename(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + filename_ = value; + onChanged(); + return this; + } + + /** + * required string filename = 2; + */ + public Builder clearFilename() { + bitField0_ = (bitField0_ & ~0x00000002); + filename_ = getDefaultInstance().getFilename(); + onChanged(); + return this; + } + + /** + * required string filename = 2; + */ + public Builder setFilenameBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + filename_ = value; + onChanged(); + return this; + } + + private long count_; + + /** + * required int64 count = 3; + */ + public boolean hasCount() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * required int64 count = 3; + */ + public long getCount() { + return count_; + } + + /** + * required int64 count = 3; + */ + public Builder setCount(long value) { + bitField0_ |= 0x00000004; + count_ = value; + onChanged(); + return this; + } + + /** + * required int64 count = 3; + */ + public Builder clearCount() { + bitField0_ = (bitField0_ & ~0x00000004); + count_ = 0L; + onChanged(); + return this; + } + + private long offset_; + + /** + * required int64 offset = 4; + */ + public boolean hasOffset() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * required int64 offset = 4; + */ + public long getOffset() { + return offset_; + } + + /** + * required int64 offset = 4; + */ + public Builder setOffset(long value) { + bitField0_ |= 0x00000008; + offset_ = value; + onChanged(); + return this; + } + + /** + * required int64 offset = 4; + */ + public Builder clearOffset() { + bitField0_ = (bitField0_ & ~0x00000008); + offset_ = 0L; + onChanged(); + return this; + } + + private boolean readPartly_; + + /** + * optional bool read_partly = 5; + */ + public boolean hasReadPartly() { + return ((bitField0_ & 0x00000010) == 0x00000010); + } + + /** + * optional bool read_partly = 5; + */ + public boolean getReadPartly() { + return readPartly_; + } + + /** + * optional bool read_partly = 5; + */ + public Builder setReadPartly(boolean value) { + bitField0_ |= 0x00000010; + readPartly_ = value; + onChanged(); + return this; + } + + /** + * optional bool read_partly = 5; + */ + public Builder clearReadPartly() { + bitField0_ = (bitField0_ & ~0x00000010); + readPartly_ = false; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetFileRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.GetFileRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetFileRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetFileRequest(input, + extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface GetFileResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.GetFileResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * 

+         * Data is in attachment
+         *
+ * + * required bool eof = 1; + */ + boolean hasEof(); + + /** + * 
+         * Data is in attachment
+         *
+ * + * required bool eof = 1; + */ + boolean getEof(); + + /** + * required bytes data = 2; + */ + boolean hasData(); + + /** + * required bytes data = 2; + */ + com.google.protobuf.ByteString getData(); + + /** + * optional int64 read_size = 3; + */ + boolean hasReadSize(); + + /** + * optional int64 read_size = 3; + */ + long getReadSize(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.GetFileResponse} + */ + public static final class GetFileResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.GetFileResponse) + GetFileResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use GetFileResponse.newBuilder() to construct. + private GetFileResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private GetFileResponse() { + eof_ = false; + data_ = com.google.protobuf.ByteString.EMPTY; + readSize_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private GetFileResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + eof_ = input.readBool(); + break; + } + case 18: { + bitField0_ |= 0x00000002; + data_ = input.readBytes(); + break; + } + case 24: { + bitField0_ |= 0x00000004; + readSize_ = input.readInt64(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000008) == 0x00000008)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000008; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.Builder.class); + } + + private int bitField0_; + public static final int EOF_FIELD_NUMBER = 1; + private boolean eof_; + + /** + * 
+         * Data is in attachment
+         *
+ * + * required bool eof = 1; + */ + public boolean hasEof() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * 
+         * Data is in attachment
+         *
+ * + * required bool eof = 1; + */ + public boolean getEof() { + return eof_; + } + + public static final int DATA_FIELD_NUMBER = 2; + private com.google.protobuf.ByteString data_; + + /** + * required bytes data = 2; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bytes data = 2; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + public static final int READ_SIZE_FIELD_NUMBER = 3; + private long readSize_; + + /** + * optional int64 read_size = 3; + */ + public boolean hasReadSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional int64 read_size = 3; + */ + public long getReadSize() { + return readSize_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasEof()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasData()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBool(1, eof_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBytes(2, data_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeInt64(3, readSize_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(1, eof_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBytesSize(2, data_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(3, readSize_); + } + if (((bitField0_ & 0x00000008) == 0x00000008)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse) obj; + + boolean result = true; + result = result && (hasEof() == other.hasEof()); + if (hasEof()) { + result = result && (getEof() == other.getEof()); + } + result = result && (hasData() == other.hasData()); + if (hasData()) { + result = result && getData().equals(other.getData()); + } + result = result && (hasReadSize() == other.hasReadSize()); + if (hasReadSize()) { + result = result && (getReadSize() == other.getReadSize()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasEof()) { + hash = (37 * hash) + EOF_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getEof()); + } + if (hasData()) { + hash = (37 * hash) + DATA_FIELD_NUMBER; + hash = (53 * hash) + getData().hashCode(); + } + if (hasReadSize()) { + hash = (37 * hash) + READ_SIZE_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getReadSize()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.GetFileResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.GetFileResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + eof_ = false; + bitField0_ = (bitField0_ & ~0x00000001); + data_ = com.google.protobuf.ByteString.EMPTY; + bitField0_ = (bitField0_ & ~0x00000002); + readSize_ = 0L; + bitField0_ = (bitField0_ & ~0x00000004); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_GetFileResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.eof_ = eof_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.data_ = data_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + result.readSize_ = readSize_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000008; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse.getDefaultInstance()) + return this; + if (other.hasEof()) { + setEof(other.getEof()); + } + if (other.hasData()) { + setData(other.getData()); + } + if (other.hasReadSize()) { + setReadSize(other.getReadSize()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasEof()) { + return false; + } + if (!hasData()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private boolean eof_; + + /** + * 
+             * Data is in attachment
+             *
+ * + * required bool eof = 1; + */ + public boolean hasEof() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * 
+             * Data is in attachment
+             *
+ * + * required bool eof = 1; + */ + public boolean getEof() { + return eof_; + } + + /** + * 
+             * Data is in attachment
+             *
+ * + * required bool eof = 1; + */ + public Builder setEof(boolean value) { + bitField0_ |= 0x00000001; + eof_ = value; + onChanged(); + return this; + } + + /** + * 
+             * Data is in attachment
+             *
+ * + * required bool eof = 1; + */ + public Builder clearEof() { + bitField0_ = (bitField0_ & ~0x00000001); + eof_ = false; + onChanged(); + return this; + } + + private com.google.protobuf.ByteString data_ = com.google.protobuf.ByteString.EMPTY; + + /** + * required bytes data = 2; + */ + public boolean hasData() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bytes data = 2; + */ + public com.google.protobuf.ByteString getData() { + return data_; + } + + /** + * required bytes data = 2; + */ + public Builder setData(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + data_ = value; + onChanged(); + return this; + } + + /** + * required bytes data = 2; + */ + public Builder clearData() { + bitField0_ = (bitField0_ & ~0x00000002); + data_ = getDefaultInstance().getData(); + onChanged(); + return this; + } + + private long readSize_; + + /** + * optional int64 read_size = 3; + */ + public boolean hasReadSize() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional int64 read_size = 3; + */ + public long getReadSize() { + return readSize_; + } + + /** + * optional int64 read_size = 3; + */ + public Builder setReadSize(long value) { + bitField0_ |= 0x00000004; + readSize_ = value; + onChanged(); + return this; + } + + /** + * optional int64 read_size = 3; + */ + public Builder clearReadSize() { + bitField0_ = (bitField0_ & ~0x00000004); + readSize_ = 0L; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000008) == 0x00000008) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000008; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000008; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.GetFileResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.GetFileResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public GetFileResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new GetFileResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ReadIndexRequestOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ReadIndexRequest) + com.google.protobuf.MessageOrBuilder { + + /** + * required string group_id = 1; + */ + boolean hasGroupId(); + + /** + * required string group_id = 1; + */ + java.lang.String getGroupId(); + + /** + * required string group_id = 1; + */ + com.google.protobuf.ByteString getGroupIdBytes(); + + /** + * required string server_id = 2; + */ + boolean hasServerId(); + + /** + * required string server_id = 2; + */ + java.lang.String getServerId(); + + /** + * required string server_id = 2; + */ + com.google.protobuf.ByteString getServerIdBytes(); + + /** + * repeated bytes entries = 3; + */ + java.util.List getEntriesList(); + + /** + * repeated bytes entries = 3; + */ + int getEntriesCount(); + + /** + * repeated bytes entries = 3; + */ + com.google.protobuf.ByteString getEntries(int index); + + /** + * optional string peer_id = 4; + */ + boolean hasPeerId(); + + /** + * optional string peer_id = 4; + */ + java.lang.String getPeerId(); + + /** + * optional string peer_id = 4; + */ + com.google.protobuf.ByteString getPeerIdBytes(); + } + + /** + * Protobuf type {@code jraft.ReadIndexRequest} + */ + public static final class ReadIndexRequest extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ReadIndexRequest) + ReadIndexRequestOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ReadIndexRequest.newBuilder() to construct. + private ReadIndexRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ReadIndexRequest() { + groupId_ = ""; + serverId_ = ""; + entries_ = java.util.Collections.emptyList(); + peerId_ = ""; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ReadIndexRequest(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 10: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000001; + groupId_ = bs; + break; + } + case 18: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000002; + serverId_ = bs; + break; + } + case 26: { + if (!((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + entries_ = new java.util.ArrayList(); + mutable_bitField0_ |= 0x00000004; + } + entries_.add(input.readBytes()); + break; + } + case 34: { + com.google.protobuf.ByteString bs = input.readBytes(); + bitField0_ |= 0x00000004; + peerId_ = bs; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + if (((mutable_bitField0_ & 0x00000004) == 0x00000004)) { + entries_ = java.util.Collections.unmodifiableList(entries_); + } + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.Builder.class); + } + + private int bitField0_; + public static final int GROUP_ID_FIELD_NUMBER = 1; + private volatile java.lang.Object groupId_; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int SERVER_ID_FIELD_NUMBER = 2; + private volatile java.lang.Object serverId_; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int ENTRIES_FIELD_NUMBER = 3; + private java.util.List entries_; + + /** + * repeated bytes entries = 3; + */ + public java.util.List getEntriesList() { + return entries_; + } + + /** + * repeated bytes entries = 3; + */ + public int getEntriesCount() { + return entries_.size(); + } + + /** + * repeated bytes entries = 3; + */ + public com.google.protobuf.ByteString getEntries(int index) { + return entries_.get(index); + } + + public static final int PEER_ID_FIELD_NUMBER = 4; + private volatile java.lang.Object peerId_; + + /** + * optional string peer_id = 4; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional string peer_id = 4; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } + } + + /** + * optional string peer_id = 4; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasGroupId()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasServerId()) { + memoizedIsInitialized = 0; + return false; + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 2, serverId_); + } + for (int i = 0; i < entries_.size(); i++) { + output.writeBytes(3, entries_.get(i)); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 4, peerId_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(1, groupId_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(2, serverId_); + } + { + int dataSize = 0; + for (int i = 0; i < entries_.size(); i++) { + dataSize += com.google.protobuf.CodedOutputStream.computeBytesSizeNoTag(entries_.get(i)); + } + size += dataSize; + size += 1 * getEntriesList().size(); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, peerId_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest other = (com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest) obj; + + boolean result = true; + result = result && (hasGroupId() == other.hasGroupId()); + if (hasGroupId()) { + result = result && getGroupId().equals(other.getGroupId()); + } + result = result && (hasServerId() == other.hasServerId()); + if (hasServerId()) { + result = result && getServerId().equals(other.getServerId()); + } + result = result && getEntriesList().equals(other.getEntriesList()); + result = result && (hasPeerId() == other.hasPeerId()); + if (hasPeerId()) { + result = result && getPeerId().equals(other.getPeerId()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasGroupId()) { + hash = (37 * hash) + GROUP_ID_FIELD_NUMBER; + hash = (53 * hash) + getGroupId().hashCode(); + } + if (hasServerId()) { + hash = (37 * hash) + SERVER_ID_FIELD_NUMBER; + hash = (53 * hash) + getServerId().hashCode(); + } + if (getEntriesCount() > 0) { + hash = (37 * hash) + ENTRIES_FIELD_NUMBER; + hash = (53 * hash) + getEntriesList().hashCode(); + } + if (hasPeerId()) { + hash = (37 * hash) + PEER_ID_FIELD_NUMBER; + hash = (53 * hash) + getPeerId().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ReadIndexRequest} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ReadIndexRequest) + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequestOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexRequest_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexRequest_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + } + } + + public Builder clear() { + super.clear(); + groupId_ = ""; + bitField0_ = (bitField0_ & ~0x00000001); + serverId_ = ""; + bitField0_ = (bitField0_ & ~0x00000002); + entries_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + peerId_ = ""; + bitField0_ = (bitField0_ & ~0x00000008); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexRequest_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest build() { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest result = new com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.groupId_ = groupId_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.serverId_ = serverId_; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + entries_ = java.util.Collections.unmodifiableList(entries_); + bitField0_ = (bitField0_ & ~0x00000004); + } + result.entries_ = entries_; + if (((from_bitField0_ & 0x00000008) == 0x00000008)) { + to_bitField0_ |= 0x00000004; + } + result.peerId_ = peerId_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest.getDefaultInstance()) + return this; + if (other.hasGroupId()) { + bitField0_ |= 0x00000001; + groupId_ = other.groupId_; + onChanged(); + } + if (other.hasServerId()) { + bitField0_ |= 0x00000002; + serverId_ = other.serverId_; + onChanged(); + } + if (!other.entries_.isEmpty()) { + if (entries_.isEmpty()) { + entries_ = other.entries_; + bitField0_ = (bitField0_ & ~0x00000004); + } else { + ensureEntriesIsMutable(); + entries_.addAll(other.entries_); + } + onChanged(); + } + if (other.hasPeerId()) { + bitField0_ |= 0x00000008; + peerId_ = other.peerId_; + onChanged(); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasGroupId()) { + return false; + } + if (!hasServerId()) { + return false; + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private java.lang.Object groupId_ = ""; + + /** + * required string group_id = 1; + */ + public boolean hasGroupId() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required string group_id = 1; + */ + public java.lang.String getGroupId() { + java.lang.Object ref = groupId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + groupId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string group_id = 1; + */ + public com.google.protobuf.ByteString getGroupIdBytes() { + java.lang.Object ref = groupId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + groupId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string group_id = 1; + */ + public Builder setGroupId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder clearGroupId() { + bitField0_ = (bitField0_ & ~0x00000001); + groupId_ = getDefaultInstance().getGroupId(); + onChanged(); + return this; + } + + /** + * required string group_id = 1; + */ + public Builder setGroupIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000001; + groupId_ = value; + onChanged(); + return this; + } + + private java.lang.Object serverId_ = ""; + + /** + * required string server_id = 2; + */ + public boolean hasServerId() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required string server_id = 2; + */ + public java.lang.String getServerId() { + java.lang.Object ref = serverId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + serverId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * required string server_id = 2; + */ + public com.google.protobuf.ByteString getServerIdBytes() { + java.lang.Object ref = serverId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + serverId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * required string server_id = 2; + */ + public Builder setServerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder clearServerId() { + bitField0_ = (bitField0_ & ~0x00000002); + serverId_ = getDefaultInstance().getServerId(); + onChanged(); + return this; + } + + /** + * required string server_id = 2; + */ + public Builder setServerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000002; + serverId_ = value; + onChanged(); + return this; + } + + private java.util.List entries_ = java.util.Collections.emptyList(); + + private void ensureEntriesIsMutable() { + if (!((bitField0_ & 0x00000004) == 0x00000004)) { + entries_ = new java.util.ArrayList(entries_); + bitField0_ |= 0x00000004; + } + } + + /** + * repeated bytes entries = 3; + */ + public java.util.List getEntriesList() { + return java.util.Collections.unmodifiableList(entries_); + } + + /** + * repeated bytes entries = 3; + */ + public int getEntriesCount() { + return entries_.size(); + } + + /** + * repeated bytes entries = 3; + */ + public com.google.protobuf.ByteString getEntries(int index) { + return entries_.get(index); + } + + /** + * repeated bytes entries = 3; + */ + public Builder setEntries(int index, com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntriesIsMutable(); + entries_.set(index, value); + onChanged(); + return this; + } + + /** + * repeated bytes entries = 3; + */ + public Builder addEntries(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + ensureEntriesIsMutable(); + entries_.add(value); + onChanged(); + return this; + } + + /** + * repeated bytes entries = 3; + */ + public Builder addAllEntries(java.lang.Iterable values) { + ensureEntriesIsMutable(); + com.google.protobuf.AbstractMessageLite.Builder.addAll(values, entries_); + onChanged(); + return this; + } + + /** + * repeated bytes entries = 3; + */ + public Builder clearEntries() { + entries_ = java.util.Collections.emptyList(); + bitField0_ = (bitField0_ & ~0x00000004); + onChanged(); + return this; + } + + private java.lang.Object peerId_ = ""; + + /** + * optional string peer_id = 4; + */ + public boolean hasPeerId() { + return ((bitField0_ & 0x00000008) == 0x00000008); + } + + /** + * optional string peer_id = 4; + */ + public java.lang.String getPeerId() { + java.lang.Object ref = peerId_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + if (bs.isValidUtf8()) { + peerId_ = s; + } + return s; + } else { + return (java.lang.String) ref; + } + } + + /** + * optional string peer_id = 4; + */ + public com.google.protobuf.ByteString getPeerIdBytes() { + java.lang.Object ref = peerId_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = com.google.protobuf.ByteString + .copyFromUtf8((java.lang.String) ref); + peerId_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * optional string peer_id = 4; + */ + public Builder setPeerId(java.lang.String value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + peerId_ = value; + onChanged(); + return this; + } + + /** + * optional string peer_id = 4; + */ + public Builder clearPeerId() { + bitField0_ = (bitField0_ & ~0x00000008); + peerId_ = getDefaultInstance().getPeerId(); + onChanged(); + return this; + } + + /** + * optional string peer_id = 4; + */ + public Builder setPeerIdBytes(com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + bitField0_ |= 0x00000008; + peerId_ = value; + onChanged(); + return this; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ReadIndexRequest) + } + + // @@protoc_insertion_point(class_scope:jraft.ReadIndexRequest) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ReadIndexRequest parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ReadIndexRequest( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + public interface ReadIndexResponseOrBuilder extends + // @@protoc_insertion_point(interface_extends:jraft.ReadIndexResponse) + com.google.protobuf.MessageOrBuilder { + + /** + * required int64 index = 1; + */ + boolean hasIndex(); + + /** + * required int64 index = 1; + */ + long getIndex(); + + /** + * required bool success = 2; + */ + boolean hasSuccess(); + + /** + * required bool success = 2; + */ + boolean getSuccess(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + boolean hasErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse(); + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder(); + } + + /** + * Protobuf type {@code jraft.ReadIndexResponse} + */ + public static final class ReadIndexResponse extends com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:jraft.ReadIndexResponse) + ReadIndexResponseOrBuilder { + private static final long serialVersionUID = 0L; + + // Use ReadIndexResponse.newBuilder() to construct. + private ReadIndexResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private ReadIndexResponse() { + index_ = 0L; + success_ = false; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet getUnknownFields() { + return this.unknownFields; + } + + private ReadIndexResponse(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet + .newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + index_ = input.readInt64(); + break; + } + case 16: { + bitField0_ |= 0x00000002; + success_ = input.readBool(); + break; + } + case 794: { + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder subBuilder = null; + if (((bitField0_ & 0x00000004) == 0x00000004)) { + subBuilder = errorResponse_.toBuilder(); + } + errorResponse_ = input.readMessage( + com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.PARSER, extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(errorResponse_); + errorResponse_ = subBuilder.buildPartial(); + } + bitField0_ |= 0x00000004; + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.Builder.class); + } + + private int bitField0_; + public static final int INDEX_FIELD_NUMBER = 1; + private long index_; + + /** + * required int64 index = 1; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 index = 1; + */ + public long getIndex() { + return index_; + } + + public static final int SUCCESS_FIELD_NUMBER = 2; + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + public static final int ERRORRESPONSE_FIELD_NUMBER = 99; + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance() + : errorResponse_; + } + + private byte memoizedIsInitialized = -1; + + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) + return true; + if (isInitialized == 0) + return false; + + if (!hasIndex()) { + memoizedIsInitialized = 0; + return false; + } + if (!hasSuccess()) { + memoizedIsInitialized = 0; + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + memoizedIsInitialized = 0; + return false; + } + } + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException { + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeInt64(1, index_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + output.writeBool(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + output.writeMessage(99, getErrorResponse()); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) + return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, index_); + } + if (((bitField0_ & 0x00000002) == 0x00000002)) { + size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, success_); + } + if (((bitField0_ & 0x00000004) == 0x00000004)) { + size += com.google.protobuf.CodedOutputStream.computeMessageSize(99, getErrorResponse()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse)) { + return super.equals(obj); + } + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse other = (com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse) obj; + + boolean result = true; + result = result && (hasIndex() == other.hasIndex()); + if (hasIndex()) { + result = result && (getIndex() == other.getIndex()); + } + result = result && (hasSuccess() == other.hasSuccess()); + if (hasSuccess()) { + result = result && (getSuccess() == other.getSuccess()); + } + result = result && (hasErrorResponse() == other.hasErrorResponse()); + if (hasErrorResponse()) { + result = result && getErrorResponse().equals(other.getErrorResponse()); + } + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasIndex()) { + hash = (37 * hash) + INDEX_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getIndex()); + } + if (hasSuccess()) { + hash = (37 * hash) + SUCCESS_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getSuccess()); + } + if (hasErrorResponse()) { + hash = (37 * hash) + ERRORRESPONSE_FIELD_NUMBER; + hash = (53 * hash) + getErrorResponse().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseDelimitedFrom(java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input, + extensionRegistry); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parseFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + public Builder toBuilder() { + return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code jraft.ReadIndexResponse} + */ + public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder + implements + // @@protoc_insertion_point(builder_implements:jraft.ReadIndexResponse) + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponseOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexResponse_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internalGetFieldAccessorTable() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexResponse_fieldAccessorTable + .ensureFieldAccessorsInitialized(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.class, + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.Builder.class); + } + + // Construct using com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder(com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) { + getErrorResponseFieldBuilder(); + } + } + + public Builder clear() { + super.clear(); + index_ = 0L; + bitField0_ = (bitField0_ & ~0x00000001); + success_ = false; + bitField0_ = (bitField0_ & ~0x00000002); + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.internal_static_jraft_ReadIndexResponse_descriptor; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse getDefaultInstanceForType() { + return com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.getDefaultInstance(); + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse build() { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse buildPartial() { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse result = new com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse( + this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.index_ = index_; + if (((from_bitField0_ & 0x00000002) == 0x00000002)) { + to_bitField0_ |= 0x00000002; + } + result.success_ = success_; + if (((from_bitField0_ & 0x00000004) == 0x00000004)) { + to_bitField0_ |= 0x00000004; + } + if (errorResponseBuilder_ == null) { + result.errorResponse_ = errorResponse_; + } else { + result.errorResponse_ = errorResponseBuilder_.build(); + } + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + + public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, java.lang.Object value) { + return (Builder) super.setField(field, value); + } + + public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + + public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + + public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index, + java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + + public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse) { + return mergeFrom((com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse other) { + if (other == com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse.getDefaultInstance()) + return this; + if (other.hasIndex()) { + setIndex(other.getIndex()); + } + if (other.hasSuccess()) { + setSuccess(other.getSuccess()); + } + if (other.hasErrorResponse()) { + mergeErrorResponse(other.getErrorResponse()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + if (!hasIndex()) { + return false; + } + if (!hasSuccess()) { + return false; + } + if (hasErrorResponse()) { + if (!getErrorResponse().isInitialized()) { + return false; + } + } + return true; + } + + public Builder mergeFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private long index_; + + /** + * required int64 index = 1; + */ + public boolean hasIndex() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + + /** + * required int64 index = 1; + */ + public long getIndex() { + return index_; + } + + /** + * required int64 index = 1; + */ + public Builder setIndex(long value) { + bitField0_ |= 0x00000001; + index_ = value; + onChanged(); + return this; + } + + /** + * required int64 index = 1; + */ + public Builder clearIndex() { + bitField0_ = (bitField0_ & ~0x00000001); + index_ = 0L; + onChanged(); + return this; + } + + private boolean success_; + + /** + * required bool success = 2; + */ + public boolean hasSuccess() { + return ((bitField0_ & 0x00000002) == 0x00000002); + } + + /** + * required bool success = 2; + */ + public boolean getSuccess() { + return success_; + } + + /** + * required bool success = 2; + */ + public Builder setSuccess(boolean value) { + bitField0_ |= 0x00000002; + success_ = value; + onChanged(); + return this; + } + + /** + * required bool success = 2; + */ + public Builder clearSuccess() { + bitField0_ = (bitField0_ & ~0x00000002); + success_ = false; + onChanged(); + return this; + } + + private com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse errorResponse_ = null; + private com.google.protobuf.SingleFieldBuilderV3 errorResponseBuilder_; + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public boolean hasErrorResponse() { + return ((bitField0_ & 0x00000004) == 0x00000004); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse getErrorResponse() { + if (errorResponseBuilder_ == null) { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } else { + return errorResponseBuilder_.getMessage(); + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + errorResponse_ = value; + onChanged(); + } else { + errorResponseBuilder_.setMessage(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder setErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder builderForValue) { + if (errorResponseBuilder_ == null) { + errorResponse_ = builderForValue.build(); + onChanged(); + } else { + errorResponseBuilder_.setMessage(builderForValue.build()); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder mergeErrorResponse(com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse value) { + if (errorResponseBuilder_ == null) { + if (((bitField0_ & 0x00000004) == 0x00000004) && errorResponse_ != null + && errorResponse_ != com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.getDefaultInstance()) { + errorResponse_ = com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.newBuilder(errorResponse_) + .mergeFrom(value).buildPartial(); + } else { + errorResponse_ = value; + } + onChanged(); + } else { + errorResponseBuilder_.mergeFrom(value); + } + bitField0_ |= 0x00000004; + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public Builder clearErrorResponse() { + if (errorResponseBuilder_ == null) { + errorResponse_ = null; + onChanged(); + } else { + errorResponseBuilder_.clear(); + } + bitField0_ = (bitField0_ & ~0x00000004); + return this; + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse.Builder getErrorResponseBuilder() { + bitField0_ |= 0x00000004; + onChanged(); + return getErrorResponseFieldBuilder().getBuilder(); + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + public com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponseOrBuilder getErrorResponseOrBuilder() { + if (errorResponseBuilder_ != null) { + return errorResponseBuilder_.getMessageOrBuilder(); + } else { + return errorResponse_ == null ? com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse + .getDefaultInstance() : errorResponse_; + } + } + + /** + * optional .jraft.ErrorResponse errorResponse = 99; + */ + private com.google.protobuf.SingleFieldBuilderV3 getErrorResponseFieldBuilder() { + if (errorResponseBuilder_ == null) { + errorResponseBuilder_ = new com.google.protobuf.SingleFieldBuilderV3( + getErrorResponse(), getParentForChildren(), isClean()); + errorResponse_ = null; + } + return errorResponseBuilder_; + } + + public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:jraft.ReadIndexResponse) + } + + // @@protoc_insertion_point(class_scope:jraft.ReadIndexResponse) + private static final com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse(); + } + + public static com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + @java.lang.Deprecated + public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { + public ReadIndexResponse parsePartialFrom(com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new ReadIndexResponse( + input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_PingRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_PingRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ErrorResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ErrorResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_InstallSnapshotRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_InstallSnapshotRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_InstallSnapshotResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_InstallSnapshotResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_TimeoutNowRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_TimeoutNowRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_TimeoutNowResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_TimeoutNowResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_RequestVoteRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_RequestVoteRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_RequestVoteResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_RequestVoteResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AppendEntriesRequestHeader_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AppendEntriesRequestHeader_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AppendEntriesRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AppendEntriesRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_AppendEntriesResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_AppendEntriesResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetFileRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetFileRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_GetFileResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetFileResponse_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ReadIndexRequest_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ReadIndexRequest_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor internal_static_jraft_ReadIndexResponse_descriptor; + private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ReadIndexResponse_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor descriptor; + static { + java.lang.String[] descriptorData = { "\n\trpc.proto\022\005jraft\032\nraft.proto\"%\n\013PingRe" + + "quest\022\026\n\016send_timestamp\030\001 \002(\003\"4\n\rErrorRe" + + "sponse\022\021\n\terrorCode\030\001 \002(\005\022\020\n\010errorMsg\030\002 " + + "\001(\t\"\214\001\n\026InstallSnapshotRequest\022\020\n\010group_" + + "id\030\001 \002(\t\022\021\n\tserver_id\030\002 \002(\t\022\017\n\007peer_id\030\003" + + " \002(\t\022\014\n\004term\030\004 \002(\003\022!\n\004meta\030\005 \002(\0132\023.jraft" + + ".SnapshotMeta\022\013\n\003uri\030\006 \002(\t\"e\n\027InstallSna" + + "pshotResponse\022\014\n\004term\030\001 \002(\003\022\017\n\007success\030\002" + + " \002(\010\022+\n\rerrorResponse\030c \001(\0132\024.jraft.Erro" + + "rResponse\"W\n\021TimeoutNowRequest\022\020\n\010group_" + + "id\030\001 \002(\t\022\021\n\tserver_id\030\002 \002(\t\022\017\n\007peer_id\030\003" + + " \002(\t\022\014\n\004term\030\004 \002(\003\"`\n\022TimeoutNowResponse" + + "\022\014\n\004term\030\001 \002(\003\022\017\n\007success\030\002 \002(\010\022+\n\rerror" + + "Response\030c \001(\0132\024.jraft.ErrorResponse\"\231\001\n" + + "\022RequestVoteRequest\022\020\n\010group_id\030\001 \002(\t\022\021\n" + + "\tserver_id\030\002 \002(\t\022\017\n\007peer_id\030\003 \002(\t\022\014\n\004ter" + + "m\030\004 \002(\003\022\025\n\rlast_log_term\030\005 \002(\003\022\026\n\016last_l" + + "og_index\030\006 \002(\003\022\020\n\010pre_vote\030\007 \002(\010\"a\n\023Requ" + + "estVoteResponse\022\014\n\004term\030\001 \002(\003\022\017\n\007granted" + + "\030\002 \002(\010\022+\n\rerrorResponse\030c \001(\0132\024.jraft.Er" + + "rorResponse\"R\n\032AppendEntriesRequestHeade" + + "r\022\020\n\010group_id\030\001 \002(\t\022\021\n\tserver_id\030\002 \002(\t\022\017" + + "\n\007peer_id\030\003 \002(\t\"\323\001\n\024AppendEntriesRequest" + + "\022\020\n\010group_id\030\001 \002(\t\022\021\n\tserver_id\030\002 \002(\t\022\017\n" + + "\007peer_id\030\003 \002(\t\022\014\n\004term\030\004 \002(\003\022\025\n\rprev_log" + + "_term\030\005 \002(\003\022\026\n\016prev_log_index\030\006 \002(\003\022!\n\007e" + + "ntries\030\007 \003(\0132\020.jraft.EntryMeta\022\027\n\017commit" + + "ted_index\030\010 \002(\003\022\014\n\004data\030\t \001(\014\"{\n\025AppendE" + + "ntriesResponse\022\014\n\004term\030\001 \002(\003\022\017\n\007success\030" + + "\002 \002(\010\022\026\n\016last_log_index\030\003 \001(\003\022+\n\rerrorRe" + + "sponse\030c \001(\0132\024.jraft.ErrorResponse\"i\n\016Ge" + + "tFileRequest\022\021\n\treader_id\030\001 \002(\003\022\020\n\010filen" + + "ame\030\002 \002(\t\022\r\n\005count\030\003 \002(\003\022\016\n\006offset\030\004 \002(\003" + + "\022\023\n\013read_partly\030\005 \001(\010\"l\n\017GetFileResponse" + + "\022\013\n\003eof\030\001 \002(\010\022\014\n\004data\030\002 \002(\014\022\021\n\tread_size" + + "\030\003 \001(\003\022+\n\rerrorResponse\030c \001(\0132\024.jraft.Er" + + "rorResponse\"Y\n\020ReadIndexRequest\022\020\n\010group" + + "_id\030\001 \002(\t\022\021\n\tserver_id\030\002 \002(\t\022\017\n\007entries\030" + + "\003 \003(\014\022\017\n\007peer_id\030\004 \001(\t\"`\n\021ReadIndexRespo" + + "nse\022\r\n\005index\030\001 \002(\003\022\017\n\007success\030\002 \002(\010\022+\n\re" + + "rrorResponse\030c \001(\0132\024.jraft.ErrorResponse" + + "B(\n\031com.alipay.sofa.jraft.rpcB\013RpcReques" + "ts" }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { com.alipay.sofa.jraft.entity.RaftOutter + .getDescriptor(), }, assigner); + internal_static_jraft_PingRequest_descriptor = getDescriptor().getMessageTypes().get(0); + internal_static_jraft_PingRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_PingRequest_descriptor, new java.lang.String[] { "SendTimestamp", }); + internal_static_jraft_ErrorResponse_descriptor = getDescriptor().getMessageTypes().get(1); + internal_static_jraft_ErrorResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ErrorResponse_descriptor, new java.lang.String[] { "ErrorCode", "ErrorMsg", }); + internal_static_jraft_InstallSnapshotRequest_descriptor = getDescriptor().getMessageTypes().get(2); + internal_static_jraft_InstallSnapshotRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_InstallSnapshotRequest_descriptor, new java.lang.String[] { "GroupId", "ServerId", + "PeerId", "Term", "Meta", "Uri", }); + internal_static_jraft_InstallSnapshotResponse_descriptor = getDescriptor().getMessageTypes().get(3); + internal_static_jraft_InstallSnapshotResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_InstallSnapshotResponse_descriptor, new java.lang.String[] { "Term", "Success", + "ErrorResponse", }); + internal_static_jraft_TimeoutNowRequest_descriptor = getDescriptor().getMessageTypes().get(4); + internal_static_jraft_TimeoutNowRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_TimeoutNowRequest_descriptor, new java.lang.String[] { "GroupId", "ServerId", + "PeerId", "Term", }); + internal_static_jraft_TimeoutNowResponse_descriptor = getDescriptor().getMessageTypes().get(5); + internal_static_jraft_TimeoutNowResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_TimeoutNowResponse_descriptor, new java.lang.String[] { "Term", "Success", + "ErrorResponse", }); + internal_static_jraft_RequestVoteRequest_descriptor = getDescriptor().getMessageTypes().get(6); + internal_static_jraft_RequestVoteRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_RequestVoteRequest_descriptor, new java.lang.String[] { "GroupId", "ServerId", + "PeerId", "Term", "LastLogTerm", "LastLogIndex", "PreVote", }); + internal_static_jraft_RequestVoteResponse_descriptor = getDescriptor().getMessageTypes().get(7); + internal_static_jraft_RequestVoteResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_RequestVoteResponse_descriptor, new java.lang.String[] { "Term", "Granted", + "ErrorResponse", }); + internal_static_jraft_AppendEntriesRequestHeader_descriptor = getDescriptor().getMessageTypes().get(8); + internal_static_jraft_AppendEntriesRequestHeader_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AppendEntriesRequestHeader_descriptor, new java.lang.String[] { "GroupId", + "ServerId", "PeerId", }); + internal_static_jraft_AppendEntriesRequest_descriptor = getDescriptor().getMessageTypes().get(9); + internal_static_jraft_AppendEntriesRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AppendEntriesRequest_descriptor, new java.lang.String[] { "GroupId", "ServerId", + "PeerId", "Term", "PrevLogTerm", "PrevLogIndex", "Entries", "CommittedIndex", "Data", }); + internal_static_jraft_AppendEntriesResponse_descriptor = getDescriptor().getMessageTypes().get(10); + internal_static_jraft_AppendEntriesResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_AppendEntriesResponse_descriptor, new java.lang.String[] { "Term", "Success", + "LastLogIndex", "ErrorResponse", }); + internal_static_jraft_GetFileRequest_descriptor = getDescriptor().getMessageTypes().get(11); + internal_static_jraft_GetFileRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetFileRequest_descriptor, new java.lang.String[] { "ReaderId", "Filename", "Count", + "Offset", "ReadPartly", }); + internal_static_jraft_GetFileResponse_descriptor = getDescriptor().getMessageTypes().get(12); + internal_static_jraft_GetFileResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_GetFileResponse_descriptor, new java.lang.String[] { "Eof", "Data", "ReadSize", + "ErrorResponse", }); + internal_static_jraft_ReadIndexRequest_descriptor = getDescriptor().getMessageTypes().get(13); + internal_static_jraft_ReadIndexRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ReadIndexRequest_descriptor, new java.lang.String[] { "GroupId", "ServerId", + "Entries", "PeerId", }); + internal_static_jraft_ReadIndexResponse_descriptor = getDescriptor().getMessageTypes().get(14); + internal_static_jraft_ReadIndexResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_jraft_ReadIndexResponse_descriptor, new java.lang.String[] { "Index", "Success", + "ErrorResponse", }); + com.alipay.sofa.jraft.entity.RaftOutter.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosure.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosure.java new file mode 100644 index 0000000..f12cfe2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosure.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.Closure; +import com.google.protobuf.Message; + +/** + * RPC response closure. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 5:55:01 PM + * @param + */ +public interface RpcResponseClosure extends Closure { + + /** + * Called by request handler to set response. + * + * @param resp rpc response + */ + void setResponse(T resp); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosureAdapter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosureAdapter.java new file mode 100644 index 0000000..00314e6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseClosureAdapter.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.google.protobuf.Message; + +/** + * RpcResponseClosure adapter holds the response. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-29 2:30:35 PM + * @param + */ +public abstract class RpcResponseClosureAdapter implements RpcResponseClosure { + + private T resp; + + public T getResponse() { + return this.resp; + } + + @Override + public void setResponse(T resp) { + this.resp = resp; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseFactory.java new file mode 100644 index 0000000..35586ac --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcResponseFactory.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RaftError; +import com.google.protobuf.Message; + +/** + * Helper to create error response. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public interface RpcResponseFactory { + + /** + * This is a convention that if a {@link Message} contains an {@link RpcRequests.ErrorResponse} field, + * it can only be in position 99. + */ + int ERROR_RESPONSE_NUM = 99; + + /** + * Creates a RPC response from status, return OK response + * when status is null. + * + * @param parent parent message + * @param st status with response + * @return a response instance + */ + default Message newResponse(final Message parent, final Status st) { + if (st == null) { + return newResponse(parent, 0, "OK"); + } + return newResponse(parent, st.getCode(), st.getErrorMsg()); + } + + /** + * Creates an error response with parameters. + * + * @param parent parent message + * @param error error with raft info + * @param fmt message with format string + * @param args arguments referenced by the format specifiers in the format string + * @return a response instance + */ + default Message newResponse(final Message parent, final RaftError error, final String fmt, final Object... args) { + return newResponse(parent, error.getNumber(), fmt, args); + } + + /** + * Creates an error response with parameters. + * + * @param parent parent message + * @param code error code with raft info + * @param fmt message with format string + * @param args arguments referenced by the format specifiers in the format string + * @return a response instance + */ + default Message newResponse(final Message parent, final int code, final String fmt, final Object... args) { + final RpcRequests.ErrorResponse.Builder eBuilder = RpcRequests.ErrorResponse.newBuilder(); + eBuilder.setErrorCode(code); + if (fmt != null) { + eBuilder.setErrorMsg(String.format(fmt, args)); + } + return eBuilder.build(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcServer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcServer.java new file mode 100644 index 0000000..9840777 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcServer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.rpc.impl.ConnectionClosedEventListener; + +/** + * + * @author jiachun.fjc + */ +public interface RpcServer extends Lifecycle { + + /** + * Register a conn closed event listener. + * + * @param listener the event listener. + */ + void registerConnectionClosedEventListener(final ConnectionClosedEventListener listener); + + /** + * Register user processor. + * + * @param processor the user processor which has a interest + */ + void registerProcessor(final RpcProcessor processor); + + /** + * + * @return bound port + */ + int boundPort(); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcUtils.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcUtils.java new file mode 100644 index 0000000..38a421a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/RpcUtils.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc; + +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.alipay.sofa.jraft.util.Utils; + +/** + * RPC utilities + * + * @author boyan(boyan@antfin.com) + */ +public final class RpcUtils { + + private static final Logger LOG = LoggerFactory.getLogger(RpcUtils.class); + + /** + * Default jraft closure executor pool minimum size, CPUs by default. + */ + public static final int MIN_RPC_CLOSURE_EXECUTOR_POOL_SIZE = SystemPropertyUtil.getInt( + "jraft.rpc.closure.threadpool.size.min", + Utils.cpus()); + + /** + * Default jraft closure executor pool maximum size. + */ + public static final int MAX_RPC_CLOSURE_EXECUTOR_POOL_SIZE = SystemPropertyUtil.getInt( + "jraft.rpc.closure.threadpool.size.max", + Math.max(100, Utils.cpus() * 5)); + + /** + * Global thread pool to run rpc closure. + */ + private static ThreadPoolExecutor RPC_CLOSURE_EXECUTOR = ThreadPoolUtil + .newBuilder() + .poolName("JRAFT_RPC_CLOSURE_EXECUTOR") + .enableMetric(true) + .coreThreads( + MIN_RPC_CLOSURE_EXECUTOR_POOL_SIZE) + .maximumThreads( + MAX_RPC_CLOSURE_EXECUTOR_POOL_SIZE) + .keepAliveSeconds(60L) + .workQueue(new SynchronousQueue<>()) + .threadFactory( + new NamedThreadFactory( + "JRaft-Rpc-Closure-Executor-", + true)) // + .build(); + + /** + * Run closure with OK status in thread pool. + */ + public static Future runClosureInThread(final Closure done) { + if (done == null) { + return null; + } + return runClosureInThread(done, Status.OK()); + } + + /** + * Run a task in thread pool, returns the future object. + */ + public static Future runInThread(final Runnable runnable) { + return RPC_CLOSURE_EXECUTOR.submit(runnable); + } + + /** + * Run closure with status in thread pool. + */ + public static Future runClosureInThread(final Closure done, final Status status) { + if (done == null) { + return null; + } + + return runInThread(() -> { + try { + done.run(status); + } catch (final Throwable t) { + LOG.error("Fail to run done closure.", t); + } + }); + } + + /** + * Run closure with status in specified executor + */ + public static void runClosureInExecutor(final Executor executor, final Closure done, final Status status) { + if (done == null) { + return; + } + + if (executor == null) { + runClosureInThread(done, status); + return; + } + + executor.execute(() -> { + try { + done.run(status); + } catch (final Throwable t) { + LOG.error("Fail to run done closure.", t); + } + }); + } + + private RpcUtils() { + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/AbstractClientService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/AbstractClientService.java new file mode 100644 index 0000000..51466e1 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/AbstractClientService.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.InvokeTimeoutException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.ClientService; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.alipay.sofa.jraft.rpc.InvokeContext; +import com.alipay.sofa.jraft.rpc.ProtobufMsgFactory; +import com.alipay.sofa.jraft.rpc.RaftRpcFactory; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.rpc.RpcResponseFactory; +import com.alipay.sofa.jraft.rpc.RpcUtils; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.alipay.sofa.jraft.util.ThreadPoolMetricSet; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.alipay.sofa.jraft.util.Utils; +import com.google.protobuf.Descriptors; +import com.google.protobuf.Message; + +/** + * Abstract RPC client service based. + + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public abstract class AbstractClientService implements ClientService { + + protected static final Logger LOG = LoggerFactory.getLogger(AbstractClientService.class); + + static { + ProtobufMsgFactory.load(); + } + + protected volatile RpcClient rpcClient; + protected ThreadPoolExecutor rpcExecutor; + protected RpcOptions rpcOptions; + + public RpcClient getRpcClient() { + return this.rpcClient; + } + + @Override + public boolean isConnected(final Endpoint endpoint) { + final RpcClient rc = this.rpcClient; + return rc != null && isConnected(rc, endpoint); + } + + private static boolean isConnected(final RpcClient rpcClient, final Endpoint endpoint) { + return rpcClient.checkConnection(endpoint); + } + + @Override + public boolean checkConnection(final Endpoint endpoint, final boolean createIfAbsent) { + final RpcClient rc = this.rpcClient; + if (rc == null) { + throw new IllegalStateException("Client service is uninitialized."); + } + return rc.checkConnection(endpoint, createIfAbsent); + } + + @Override + public synchronized boolean init(final RpcOptions rpcOptions) { + if (this.rpcClient != null) { + return true; + } + this.rpcOptions = rpcOptions; + return initRpcClient(this.rpcOptions.getRpcProcessorThreadPoolSize()); + } + + protected void configRpcClient(final RpcClient rpcClient) { + // NO-OP + } + + protected boolean initRpcClient(final int rpcProcessorThreadPoolSize) { + final RaftRpcFactory factory = RpcFactoryHelper.rpcFactory(); + this.rpcClient = factory.createRpcClient(factory.defaultJRaftClientConfigHelper(this.rpcOptions)); + configRpcClient(this.rpcClient); + this.rpcClient.init(null); + this.rpcExecutor = ThreadPoolUtil.newBuilder() // + .poolName("JRaft-RPC-Processor") // + .enableMetric(true) // + .coreThreads(rpcProcessorThreadPoolSize / 3) // + .maximumThreads(rpcProcessorThreadPoolSize) // + .keepAliveSeconds(60L) // + .workQueue(new ArrayBlockingQueue<>(10000)) // + .threadFactory(new NamedThreadFactory("JRaft-RPC-Processor-", true)) // + .build(); + if (this.rpcOptions.getMetricRegistry() != null) { + this.rpcOptions.getMetricRegistry().register("raft-rpc-client-thread-pool", + new ThreadPoolMetricSet(this.rpcExecutor)); + Utils.registerClosureExecutorMetrics(this.rpcOptions.getMetricRegistry()); + } + return true; + } + + @Override + public synchronized void shutdown() { + if (this.rpcClient != null) { + this.rpcClient.shutdown(); + this.rpcClient = null; + this.rpcExecutor.shutdown(); + } + } + + @Override + public boolean connect(final Endpoint endpoint) { + final RpcClient rc = this.rpcClient; + if (rc == null) { + throw new IllegalStateException("Client service is uninitialized."); + } + if (isConnected(rc, endpoint)) { + return true; + } + try { + final PingRequest req = PingRequest.newBuilder() // + .setSendTimestamp(System.currentTimeMillis()) // + .build(); + final ErrorResponse resp = (ErrorResponse) rc.invokeSync(endpoint, req, + this.rpcOptions.getRpcConnectTimeoutMs()); + return resp.getErrorCode() == 0; + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } catch (final RemotingException e) { + LOG.error("Fail to connect {}, remoting exception: {}.", endpoint, e.getMessage()); + return false; + } + } + + @Override + public boolean disconnect(final Endpoint endpoint) { + final RpcClient rc = this.rpcClient; + if (rc == null) { + return true; + } + LOG.info("Disconnect from {}.", endpoint); + rc.closeConnection(endpoint); + return true; + } + + @Override + public Future invokeWithDone(final Endpoint endpoint, final Message request, + final RpcResponseClosure done, final int timeoutMs) { + return invokeWithDone(endpoint, request, done, timeoutMs, this.rpcExecutor); + } + + public Future invokeWithDone(final Endpoint endpoint, final Message request, + final RpcResponseClosure done, final int timeoutMs, + final Executor rpcExecutor) { + return invokeWithDone(endpoint, request, null, done, timeoutMs, rpcExecutor); + } + + public Future invokeWithDone(final Endpoint endpoint, final Message request, + final InvokeContext ctx, + final RpcResponseClosure done, final int timeoutMs) { + return invokeWithDone(endpoint, request, ctx, done, timeoutMs, this.rpcExecutor); + } + + public Future invokeWithDone(final Endpoint endpoint, final Message request, + final InvokeContext ctx, + final RpcResponseClosure done, final int timeoutMs, + final Executor rpcExecutor) { + final RpcClient rc = this.rpcClient; + final FutureImpl future = new FutureImpl<>(); + final Executor currExecutor = rpcExecutor != null ? rpcExecutor : this.rpcExecutor; + try { + if (rc == null) { + future.failure(new IllegalStateException("Client service is uninitialized.")); + // should be in another thread to avoid dead locking. + RpcUtils.runClosureInExecutor(currExecutor, done, new Status(RaftError.EINTERNAL, + "Client service is uninitialized.")); + return future; + } + + rc.invokeAsync(endpoint, request, ctx, new InvokeCallback() { + + @SuppressWarnings({ "unchecked", "ConstantConditions" }) + @Override + public void complete(final Object result, final Throwable err) { + if (future.isCancelled()) { + onCanceled(request, done); + return; + } + + if (err == null) { + Status status = Status.OK(); + Message msg; + if (result instanceof ErrorResponse) { + status = handleErrorResponse((ErrorResponse) result); + msg = (Message) result; + } else if (result instanceof Message) { + final Descriptors.FieldDescriptor fd = ((Message) result).getDescriptorForType() // + .findFieldByNumber(RpcResponseFactory.ERROR_RESPONSE_NUM); + if (fd != null && ((Message) result).hasField(fd)) { + final ErrorResponse eResp = (ErrorResponse) ((Message) result).getField(fd); + status = handleErrorResponse(eResp); + msg = eResp; + } else { + msg = (T) result; + } + } else { + msg = (T) result; + } + if (done != null) { + try { + if (status.isOk()) { + done.setResponse((T) msg); + } + done.run(status); + } catch (final Throwable t) { + LOG.error("Fail to run RpcResponseClosure, the request is {}.", request, t); + } + } + if (!future.isDone()) { + future.setResult(msg); + } + } else { + if (done != null) { + try { + done.run(new Status(err instanceof InvokeTimeoutException ? RaftError.ETIMEDOUT + : RaftError.EINTERNAL, "RPC exception:" + err.getMessage())); + } catch (final Throwable t) { + LOG.error("Fail to run RpcResponseClosure, the request is {}.", request, t); + } + } + if (!future.isDone()) { + future.failure(err); + } + } + } + + @Override + public Executor executor() { + return currExecutor; + } + }, timeoutMs <= 0 ? this.rpcOptions.getRpcDefaultTimeout() : timeoutMs); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + future.failure(e); + // should be in another thread to avoid dead locking. + RpcUtils.runClosureInExecutor(currExecutor, done, + new Status(RaftError.EINTR, "Sending rpc was interrupted")); + } catch (final RemotingException e) { + future.failure(e); + // should be in another thread to avoid dead locking. + RpcUtils.runClosureInExecutor(currExecutor, done, new Status(RaftError.EINTERNAL, + "Fail to send a RPC request:" + e.getMessage())); + + } + + return future; + } + + private static Status handleErrorResponse(final ErrorResponse eResp) { + final Status status = new Status(); + status.setCode(eResp.getErrorCode()); + if (eResp.hasErrorMsg()) { + status.setErrorMsg(eResp.getErrorMsg()); + } + return status; + } + + private void onCanceled(final Message request, final RpcResponseClosure done) { + if (done != null) { + try { + done.run(new Status(RaftError.ECANCELED, "RPC request was canceled by future.")); + } catch (final Throwable t) { + LOG.error("Fail to run RpcResponseClosure, the request is {}.", request, t); + } + } + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRaftRpcFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRaftRpcFactory.java new file mode 100644 index 0000000..9fda8b0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRaftRpcFactory.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.remoting.CustomSerializerManager; +import com.alipay.remoting.InvokeContext; +import com.alipay.remoting.rpc.RpcConfigManager; +import com.alipay.remoting.rpc.RpcConfigs; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.ProtobufSerializer; +import com.alipay.sofa.jraft.rpc.RaftRpcFactory; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.RpcServer; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.SPI; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; + +/** + * + * @author jiachun.fjc + */ +@SPI +public class BoltRaftRpcFactory implements RaftRpcFactory { + + private static final Logger LOG = LoggerFactory.getLogger(BoltRaftRpcFactory.class); + + static final int CHANNEL_WRITE_BUF_LOW_WATER_MARK = SystemPropertyUtil.getInt( + "bolt.channel_write_buf_low_water_mark", + 256 * 1024); + static final int CHANNEL_WRITE_BUF_HIGH_WATER_MARK = SystemPropertyUtil.getInt( + "bolt.channel_write_buf_high_water_mark", + 512 * 1024); + + @Override + public void registerProtobufSerializer(final String className, final Object... args) { + CustomSerializerManager.registerCustomSerializer(className, ProtobufSerializer.INSTANCE); + } + + @Override + public RpcClient createRpcClient(final ConfigHelper helper) { + final com.alipay.remoting.rpc.RpcClient boltImpl = new com.alipay.remoting.rpc.RpcClient(); + final RpcClient rpcClient = new BoltRpcClient(boltImpl); + if (helper != null) { + helper.config(rpcClient); + } + return rpcClient; + } + + @Override + public RpcServer createRpcServer(final Endpoint endpoint, final ConfigHelper helper) { + final int port = Requires.requireNonNull(endpoint, "endpoint").getPort(); + Requires.requireTrue(port > 0 && port < 0xFFFF, "port out of range:" + port); + final com.alipay.remoting.rpc.RpcServer boltImpl = new com.alipay.remoting.rpc.RpcServer(port, true, false); + final RpcServer rpcServer = new BoltRpcServer(boltImpl); + if (helper != null) { + helper.config(rpcServer); + } + return rpcServer; + } + + @Override + public ConfigHelper defaultJRaftClientConfigHelper(final RpcOptions opts) { + return ins -> { + final BoltRpcClient client = (BoltRpcClient) ins; + final InvokeContext ctx = new InvokeContext(); + ctx.put(InvokeContext.BOLT_CRC_SWITCH, opts.isEnableRpcChecksum()); + client.setDefaultInvokeCtx(ctx); + }; + } + + @Override + public void ensurePipeline() { + // enable `bolt.rpc.dispatch-msg-list-in-default-executor` system property + if (RpcConfigManager.dispatch_msg_list_in_default_executor()) { + System.setProperty(RpcConfigs.DISPATCH_MSG_LIST_IN_DEFAULT_EXECUTOR, "false"); + LOG.warn("JRaft SET {} to be false for replicator pipeline optimistic.", + RpcConfigs.DISPATCH_MSG_LIST_IN_DEFAULT_EXECUTOR); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcClient.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcClient.java new file mode 100644 index 0000000..9903ffe --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcClient.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import java.util.Map; +import java.util.concurrent.Executor; + +import com.alipay.remoting.ConnectionEventType; +import com.alipay.remoting.RejectedExecutionPolicy; +import com.alipay.remoting.config.BoltClientOption; +import com.alipay.remoting.config.switches.GlobalSwitch; +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.error.InvokeTimeoutException; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.InvokeCallback; +import com.alipay.sofa.jraft.rpc.InvokeContext; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.impl.core.ClientServiceConnectionEventProcessor; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.Requires; + +/** + * Bolt rpc client impl. + * + * @author jiachun.fjc + */ +public class BoltRpcClient implements RpcClient { + + public static final String BOLT_CTX = "BOLT_CTX"; + public static final String BOLT_REJECTED_EXECUTION_POLICY = "BOLT_REJECTED_EXECUTION_POLICY"; + + private final com.alipay.remoting.rpc.RpcClient rpcClient; + private com.alipay.remoting.InvokeContext defaultInvokeCtx; + + public BoltRpcClient(com.alipay.remoting.rpc.RpcClient rpcClient) { + this.rpcClient = Requires.requireNonNull(rpcClient, "rpcClient"); + } + + @Override + public boolean init(final RpcOptions opts) { + rpcClient.option(BoltClientOption.NETTY_FLUSH_CONSOLIDATION, true); + this.rpcClient.initWriteBufferWaterMark(BoltRaftRpcFactory.CHANNEL_WRITE_BUF_LOW_WATER_MARK, + BoltRaftRpcFactory.CHANNEL_WRITE_BUF_HIGH_WATER_MARK); + this.rpcClient.enableReconnectSwitch(); + this.rpcClient.startup(); + return true; + } + + @Override + public void shutdown() { + this.rpcClient.shutdown(); + } + + @Override + public boolean checkConnection(final Endpoint endpoint) { + Requires.requireNonNull(endpoint, "endpoint"); + return this.rpcClient.checkConnection(endpoint.toString()); + } + + @Override + public boolean checkConnection(final Endpoint endpoint, final boolean createIfAbsent) { + Requires.requireNonNull(endpoint, "endpoint"); + return this.rpcClient.checkConnection(endpoint.toString(), createIfAbsent, true); + } + + @Override + public void closeConnection(final Endpoint endpoint) { + Requires.requireNonNull(endpoint, "endpoint"); + this.rpcClient.closeConnection(endpoint.toString()); + } + + @Override + public void registerConnectEventListener(final ReplicatorGroup replicatorGroup) { + this.rpcClient.addConnectionEventProcessor(ConnectionEventType.CONNECT, + new ClientServiceConnectionEventProcessor(replicatorGroup)); + } + + @Override + public Object invokeSync(final Endpoint endpoint, final Object request, final InvokeContext ctx, + final long timeoutMs) throws InterruptedException, RemotingException { + Requires.requireNonNull(endpoint, "endpoint"); + try { + return this.rpcClient.invokeSync(endpoint.toString(), request, getBoltInvokeCtx(ctx), (int) timeoutMs); + } catch (final com.alipay.remoting.rpc.exception.InvokeTimeoutException e) { + throw new InvokeTimeoutException(e); + } catch (final com.alipay.remoting.exception.RemotingException e) { + throw new RemotingException(e); + } + } + + @Override + public void invokeAsync(final Endpoint endpoint, final Object request, final InvokeContext ctx, + final InvokeCallback callback, final long timeoutMs) throws InterruptedException, + RemotingException { + Requires.requireNonNull(endpoint, "endpoint"); + try { + this.rpcClient.invokeWithCallback(endpoint.toString(), request, getBoltInvokeCtx(ctx), + getBoltCallback(callback, ctx), (int) timeoutMs); + } catch (final com.alipay.remoting.rpc.exception.InvokeTimeoutException e) { + throw new InvokeTimeoutException(e); + } catch (final com.alipay.remoting.exception.RemotingException e) { + throw new RemotingException(e); + } + } + + public com.alipay.remoting.rpc.RpcClient getRpcClient() { + return rpcClient; + } + + public com.alipay.remoting.InvokeContext getDefaultInvokeCtx() { + return defaultInvokeCtx; + } + + public void setDefaultInvokeCtx(com.alipay.remoting.InvokeContext defaultInvokeCtx) { + this.defaultInvokeCtx = defaultInvokeCtx; + } + + private RejectedExecutionPolicy getRejectedPolicy(final InvokeContext ctx) { + return ctx == null ? RejectedExecutionPolicy.CALLER_HANDLE_EXCEPTION : ctx.getOrDefault( + BOLT_REJECTED_EXECUTION_POLICY, RejectedExecutionPolicy.CALLER_HANDLE_EXCEPTION); + } + + private com.alipay.remoting.InvokeContext getBoltInvokeCtx(final InvokeContext ctx) { + if (ctx == null) { + return this.defaultInvokeCtx; + } + + com.alipay.remoting.InvokeContext boltCtx = ctx.get(BOLT_CTX); + if (boltCtx != null) { + return boltCtx; + } + + boltCtx = new com.alipay.remoting.InvokeContext(); + for (Map.Entry entry : ctx.entrySet()) { + boltCtx.put(entry.getKey(), entry.getValue()); + } + final Boolean crcSwitch = ctx.get(InvokeContext.CRC_SWITCH); + if (crcSwitch != null) { + boltCtx.put(com.alipay.remoting.InvokeContext.BOLT_CRC_SWITCH, crcSwitch); + } + return boltCtx; + } + + private BoltCallback getBoltCallback(final InvokeCallback callback, final InvokeContext ctx) { + Requires.requireNonNull(callback, "callback"); + return new BoltCallback(callback, getRejectedPolicy(ctx)); + } + + private static class BoltCallback implements com.alipay.remoting.RejectionProcessableInvokeCallback { + + private final InvokeCallback callback; + private final RejectedExecutionPolicy rejectedPolicy; + + private BoltCallback(final InvokeCallback callback, final RejectedExecutionPolicy rejectedPolicy) { + this.callback = callback; + this.rejectedPolicy = rejectedPolicy; + } + + @Override + public void onResponse(final Object result) { + this.callback.complete(result, null); + } + + @Override + public void onException(final Throwable err) { + this.callback.complete(null, err); + } + + @Override + public Executor getExecutor() { + return this.callback.executor(); + } + + @Override + public RejectedExecutionPolicy rejectedExecutionPolicy() { + return this.rejectedPolicy; + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcServer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcServer.java new file mode 100644 index 0000000..ae00864 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/BoltRpcServer.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import java.util.concurrent.Executor; + +import com.alipay.remoting.AsyncContext; +import com.alipay.remoting.BizContext; +import com.alipay.remoting.ConnectionEventType; +import com.alipay.remoting.config.BoltClientOption; +import com.alipay.remoting.config.switches.GlobalSwitch; +import com.alipay.remoting.rpc.protocol.AsyncUserProcessor; +import com.alipay.sofa.jraft.rpc.Connection; +import com.alipay.sofa.jraft.rpc.RpcContext; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.alipay.sofa.jraft.rpc.RpcServer; +import com.alipay.sofa.jraft.util.Requires; + +/** + * Bolt RPC server impl. + * + * @author jiachun.fjc + */ +public class BoltRpcServer implements RpcServer { + + private final com.alipay.remoting.rpc.RpcServer rpcServer; + + public BoltRpcServer(final com.alipay.remoting.rpc.RpcServer rpcServer) { + this.rpcServer = Requires.requireNonNull(rpcServer, "rpcServer"); + } + + @Override + public boolean init(final Void opts) { + this.rpcServer.option(BoltClientOption.NETTY_FLUSH_CONSOLIDATION, true); + this.rpcServer.initWriteBufferWaterMark(BoltRaftRpcFactory.CHANNEL_WRITE_BUF_LOW_WATER_MARK, + BoltRaftRpcFactory.CHANNEL_WRITE_BUF_HIGH_WATER_MARK); + this.rpcServer.startup(); + return this.rpcServer.isStarted(); + } + + @Override + public void shutdown() { + this.rpcServer.shutdown(); + } + + @Override + public void registerConnectionClosedEventListener(final ConnectionClosedEventListener listener) { + this.rpcServer.addConnectionEventProcessor(ConnectionEventType.CLOSE, (remoteAddress, conn) -> { + final Connection proxyConn = conn == null ? null : new Connection() { + + @Override + public Object getAttribute(final String key) { + return conn.getAttribute(key); + } + + @Override + public Object setAttributeIfAbsent(final String key, final Object value) { + return conn.setAttributeIfAbsent(key, value); + } + + @Override + public void setAttribute(final String key, final Object value) { + conn.setAttribute(key, value); + } + + @Override + public void close() { + conn.close(); + } + }; + + listener.onClosed(remoteAddress, proxyConn); + }); + } + + @Override + public int boundPort() { + return this.rpcServer.port(); + } + + @Override + public void registerProcessor(final RpcProcessor processor) { + this.rpcServer.registerUserProcessor(new AsyncUserProcessor() { + + @SuppressWarnings("unchecked") + @Override + public void handleRequest(final BizContext bizCtx, final AsyncContext asyncCtx, final Object request) { + final RpcContext rpcCtx = new RpcContext() { + + @Override + public void sendResponse(final Object responseObj) { + asyncCtx.sendResponse(responseObj); + } + + @Override + public Connection getConnection() { + com.alipay.remoting.Connection conn = bizCtx.getConnection(); + if (conn == null) { + return null; + } + return new BoltConnection(conn); + } + + @Override + public String getRemoteAddress() { + return bizCtx.getRemoteAddress(); + } + }; + + processor.handleRequest(rpcCtx, request); + } + + @Override + public String interest() { + return processor.interest(); + } + + @Override + public ExecutorSelector getExecutorSelector() { + final RpcProcessor.ExecutorSelector realSelector = processor.executorSelector(); + if (realSelector == null) { + return null; + } + return realSelector::select; + } + + @Override + public Executor getExecutor() { + return processor.executor(); + } + }); + } + + public com.alipay.remoting.rpc.RpcServer getServer() { + return this.rpcServer; + } + + private static class BoltConnection implements Connection { + + private final com.alipay.remoting.Connection conn; + + private BoltConnection(final com.alipay.remoting.Connection conn) { + this.conn = Requires.requireNonNull(conn, "conn"); + } + + @Override + public Object getAttribute(final String key) { + return this.conn.getAttribute(key); + } + + @Override + public Object setAttributeIfAbsent(final String key, final Object value) { + return this.conn.setAttributeIfAbsent(key, value); + } + + @Override + public void setAttribute(final String key, final Object value) { + this.conn.setAttribute(key, value); + } + + @Override + public void close() { + this.conn.close(); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionClosedEventListener.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionClosedEventListener.java new file mode 100644 index 0000000..fe069d0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionClosedEventListener.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import com.alipay.sofa.jraft.rpc.Connection; + +/** + * + * @author jiachun.fjc + */ +public interface ConnectionClosedEventListener { + + void onClosed(final String remoteAddress, final Connection conn); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/FutureImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/FutureImpl.java new file mode 100644 index 0000000..6c7e6e7 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/FutureImpl.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +/* + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright 2007-2008 Sun Microsystems, Inc. All rights reserved. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common Development + * and Distribution License("CDDL") (collectively, the "License"). You + * may not use this file except in compliance with the License. You can obtain + * a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html + * or glassfish/bootstrap/legal/LICENSE.txt. See the License for the specific + * language governing permissions and limitations under the License. + * + * When distributing the software, include this License Header Notice in each + * file and include the License file at glassfish/bootstrap/legal/LICENSE.txt. + * Sun designates this particular file as subject to the "Classpath" exception + * as provided by Sun in the GPL Version 2 section of the License file that + * accompanied this code. If applicable, add the following below the License + * Header, with the fields enclosed by brackets [] replaced by your own + * identifying information: "Portions Copyrighted [year] + * [name of copyright owner]" + * + * Contributor(s): + * + * If you wish your version of this file to be governed by only the CDDL or + * only the GPL Version 2, indicate your decision by adding "[Contributor] + * elects to include this software in this distribution under the [CDDL or GPL + * Version 2] license." If you don't indicate a single choice of license, a + * recipient has the option to distribute your version of this file under + * either the CDDL, the GPL Version 2 or to extend the choice of license to + * its licensees as provided above. However, if you add GPL Version 2 code + * and therefore, elected the GPL Version 2 license, then the option applies + * only if the new code is made subject to such option by the copyright + * holder. + * + */ + +import java.util.concurrent.CancellationException; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Simple {@link Future} implementation, which uses {@link ReentrantLock} to + * synchronize during the lifecycle. + * + * @see Future + * @see ReentrantLock + * + * @author Alexey Stashok + */ +public class FutureImpl implements Future { + + protected final ReentrantLock lock; + + protected boolean isDone; + + protected CountDownLatch latch; + + protected boolean isCancelled; + protected Throwable failure; + + protected R result; + + public FutureImpl() { + this(new ReentrantLock()); + } + + public FutureImpl(ReentrantLock lock) { + this.lock = lock; + this.latch = new CountDownLatch(1); + } + + /** + * Get current result value without any blocking. + * + * @return current result value without any blocking. + */ + public R getResult() { + this.lock.lock(); + try { + return this.result; + } finally { + this.lock.unlock(); + } + } + + public Throwable getFailure() { + this.lock.lock(); + try { + return this.failure; + } finally { + this.lock.unlock(); + } + } + + /** + * Set the result value and notify about operation completion. + * + * @param result + * the result value + */ + public void setResult(R result) { + this.lock.lock(); + try { + this.result = result; + notifyHaveResult(); + } finally { + this.lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean cancel(boolean mayInterruptIfRunning) { + this.lock.lock(); + try { + this.isCancelled = true; + notifyHaveResult(); + return true; + } finally { + this.lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isCancelled() { + try { + this.lock.lock(); + return this.isCancelled; + } finally { + this.lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public boolean isDone() { + this.lock.lock(); + try { + return this.isDone; + } finally { + this.lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public R get() throws InterruptedException, ExecutionException { + this.latch.await(); + this.lock.lock(); + try { + if (this.isCancelled) { + throw new CancellationException(); + } else if (this.failure != null) { + throw new ExecutionException(this.failure); + } + + return this.result; + } finally { + this.lock.unlock(); + } + } + + /** + * {@inheritDoc} + */ + @Override + public R get(final long timeout, final TimeUnit unit) throws InterruptedException, ExecutionException, + TimeoutException { + final boolean isTimeOut = !latch.await(timeout, unit); + this.lock.lock(); + try { + if (!isTimeOut) { + if (this.isCancelled) { + throw new CancellationException(); + } else if (this.failure != null) { + throw new ExecutionException(this.failure); + } + + return this.result; + } else { + throw new TimeoutException(); + } + } finally { + this.lock.unlock(); + } + } + + /** + * Notify about the failure, occured during asynchronous operation + * execution. + */ + public void failure(final Throwable failure) { + this.lock.lock(); + try { + this.failure = failure; + notifyHaveResult(); + } finally { + this.lock.unlock(); + } + } + + /** + * Notify blocked listeners threads about operation completion. + */ + protected void notifyHaveResult() { + this.isDone = true; + this.latch.countDown(); + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessor.java new file mode 100644 index 0000000..89f8603 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessor.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl; + +import com.alipay.sofa.jraft.rpc.RpcContext; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; + +/** + * Ping request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class PingRequestProcessor implements RpcProcessor { + + @Override + public void handleRequest(final RpcContext rpcCtx, final PingRequest request) { + rpcCtx.sendResponse( // + RpcFactoryHelper // + .responseFactory() // + .newResponse(RpcRequests.ErrorResponse.getDefaultInstance(), 0, "OK")); + } + + @Override + public String interest() { + return PingRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessor.java new file mode 100644 index 0000000..1a5beaa --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessor.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests; +import com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * AddLearners request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class AddLearnersRequestProcessor extends BaseCliRequestProcessor { + + public AddLearnersRequestProcessor(final Executor executor) { + super(executor, CliRequests.LearnersOpResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final AddLearnersRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final AddLearnersRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final AddLearnersRequest request, + final RpcRequestClosure done) { + final List oldLearners = ctx.node.listLearners(); + final List addingLearners = new ArrayList<>(request.getLearnersCount()); + + for (final String peerStr : request.getLearnersList()) { + final PeerId peer = new PeerId(); + if (!peer.parse(peerStr)) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerStr); + } + addingLearners.add(peer); + } + + LOG.info("Receive AddLearnersRequest to {} from {}, adding {}.", ctx.node.getNodeId(), + done.getRpcCtx().getRemoteAddress(), addingLearners); + ctx.node.addLearners(addingLearners, status -> { + if (!status.isOk()) { + done.run(status); + } else { + final LearnersOpResponse.Builder rb = LearnersOpResponse.newBuilder(); + + for (final PeerId peer : oldLearners) { + rb.addOldLearners(peer.toString()); + rb.addNewLearners(peer.toString()); + } + + for (final PeerId peer : addingLearners) { + if (!oldLearners.contains(peer)) { + rb.addNewLearners(peer.toString()); + } + } + + done.sendResponse(rb.build()); + } + }); + + return null; + } + + @Override + public String interest() { + return AddLearnersRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessor.java new file mode 100644 index 0000000..87ab712 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * AddPeer request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class AddPeerRequestProcessor extends BaseCliRequestProcessor { + + public AddPeerRequestProcessor(Executor executor) { + super(executor, AddPeerResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final AddPeerRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final AddPeerRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final AddPeerRequest request, final RpcRequestClosure done) { + final List oldPeers = ctx.node.listPeers(); + final String addingPeerIdStr = request.getPeerId(); + final PeerId addingPeer = new PeerId(); + if (addingPeer.parse(addingPeerIdStr)) { + LOG.info("Receive AddPeerRequest to {} from {}, adding {}", ctx.node.getNodeId(), done.getRpcCtx() + .getRemoteAddress(), addingPeerIdStr); + ctx.node.addPeer(addingPeer, status -> { + if (!status.isOk()) { + done.run(status); + } else { + final AddPeerResponse.Builder rb = AddPeerResponse.newBuilder(); + boolean alreadyExists = false; + for (final PeerId oldPeer : oldPeers) { + rb.addOldPeers(oldPeer.toString()); + rb.addNewPeers(oldPeer.toString()); + if (oldPeer.equals(addingPeer)) { + alreadyExists = true; + } + } + if (!alreadyExists) { + rb.addNewPeers(addingPeerIdStr); + } + done.sendResponse(rb.build()); + } + }); + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", addingPeerIdStr); + } + + return null; + } + + @Override + public String interest() { + return AddPeerRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessor.java new file mode 100644 index 0000000..ba86519 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessor.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.List; +import java.util.concurrent.Executor; + +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Base template to handle cli requests. + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 11:51:42 AM + * @param + */ +public abstract class BaseCliRequestProcessor extends RpcRequestProcessor { + + protected static final Logger LOG = LoggerFactory.getLogger(BaseCliRequestProcessor.class); + + public BaseCliRequestProcessor(Executor executor, Message defaultResp) { + super(executor, defaultResp); + } + + /** + * Returns the peerId that will be find in node manager. + */ + protected abstract String getPeerId(T request); + + /** + * Returns the raft group id + */ + protected abstract String getGroupId(T request); + + /** + * Process the request with CliRequestContext + */ + protected abstract Message processRequest0(CliRequestContext ctx, T request, RpcRequestClosure done); + + /** + * Cli request context + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-09 11:55:45 AM + */ + public static class CliRequestContext { + + /** + * The found node. + */ + public final Node node; + /** + * The peerId in returns by {@link BaseCliRequestProcessor#getPeerId(Message)}, null if absent. + */ + public final PeerId peerId; + /** + * The groupId in request. + */ + public final String groupId; + + public CliRequestContext(Node ndoe, String groupId, PeerId peerId) { + super(); + this.node = ndoe; + this.peerId = peerId; + this.groupId = groupId; + } + + } + + @Override + public Message processRequest(T request, RpcRequestClosure done) { + + String groupId = getGroupId(request); + String peerIdStr = getPeerId(request); + PeerId peerId = null; + + if (!StringUtils.isBlank(peerIdStr)) { + peerId = new PeerId(); + if (!peerId.parse(peerIdStr)) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer: %s", peerIdStr); + } + } + + Status st = new Status(); + Node node = getNode(groupId, peerId, st); + if (!st.isOk()) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), st.getCode(), st.getErrorMsg()); + } else { + return processRequest0(new CliRequestContext(node, groupId, peerId), request, done); + } + } + + protected Node getNode(String groupId, PeerId peerId, Status st) { + Node node = null; + + if (peerId != null) { + node = NodeManager.getInstance().get(groupId, peerId); + if (node == null) { + st.setError(RaftError.ENOENT, "Fail to find node %s in group %s", peerId, groupId); + } + } else { + List nodes = NodeManager.getInstance().getNodesByGroupId(groupId); + if (nodes == null || nodes.isEmpty()) { + st.setError(RaftError.ENOENT, "Empty nodes in group %s", groupId); + } else if (nodes.size() > 1) { + st.setError(RaftError.EINVAL, "Peer must be specified since there're %d nodes in group %s", + nodes.size(), groupId); + } else { + node = nodes.get(0); + } + + } + if (node != null && node.getOptions().isDisableCli()) { + st.setError(RaftError.EACCES, "Cli service is not allowed to access node %s", node.getNodeId()); + } + return node; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessor.java new file mode 100644 index 0000000..2e245b6 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessor.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Change peers request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class ChangePeersRequestProcessor extends BaseCliRequestProcessor { + + public ChangePeersRequestProcessor(Executor executor) { + super(executor, ChangePeersResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final ChangePeersRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final ChangePeersRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final ChangePeersRequest request, final RpcRequestClosure done) { + final List oldConf = ctx.node.listPeers(); + + final Configuration conf = new Configuration(); + for (final String peerIdStr : request.getNewPeersList()) { + final PeerId peer = new PeerId(); + if (peer.parse(peerIdStr)) { + conf.addPeer(peer); + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerIdStr); + } + } + LOG.info("Receive ChangePeersRequest to {} from {}, new conf is {}", ctx.node.getNodeId(), done.getRpcCtx() + .getRemoteAddress(), conf); + ctx.node.changePeers(conf, status -> { + if (!status.isOk()) { + done.run(status); + } else { + ChangePeersResponse.Builder rb = ChangePeersResponse.newBuilder(); + for (final PeerId peer : oldConf) { + rb.addOldPeers(peer.toString()); + } + for (final PeerId peer : conf) { + rb.addNewPeers(peer.toString()); + } + done.sendResponse(rb.build()); + } + }); + return null; + } + + @Override + public String interest() { + return ChangePeersRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/CliClientServiceImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/CliClientServiceImpl.java new file mode 100644 index 0000000..679c963 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/CliClientServiceImpl.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.concurrent.Future; + +import com.alipay.sofa.jraft.option.CliOptions; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.CliClientService; +import com.alipay.sofa.jraft.rpc.CliRequests; +import com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.rpc.impl.AbstractClientService; +import com.alipay.sofa.jraft.util.Endpoint; +import com.google.protobuf.Message; + +/** + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class CliClientServiceImpl extends AbstractClientService implements CliClientService { + + private CliOptions cliOptions; + + @Override + public synchronized boolean init(final RpcOptions rpcOptions) { + boolean ret = super.init(rpcOptions); + if (ret) { + this.cliOptions = (CliOptions) this.rpcOptions; + } + return ret; + } + + @Override + public Future addPeer(final Endpoint endpoint, final AddPeerRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future removePeer(final Endpoint endpoint, final RemovePeerRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future resetPeer(final Endpoint endpoint, final ResetPeerRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future snapshot(final Endpoint endpoint, final SnapshotRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future changePeers(final Endpoint endpoint, final ChangePeersRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future addLearners(final Endpoint endpoint, final AddLearnersRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future removeLearners(final Endpoint endpoint, final RemoveLearnersRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future resetLearners(final Endpoint endpoint, final ResetLearnersRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future getLeader(final Endpoint endpoint, final GetLeaderRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future transferLeader(final Endpoint endpoint, final TransferLeaderRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } + + @Override + public Future getPeers(final Endpoint endpoint, final CliRequests.GetPeersRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.cliOptions.getTimeoutMs()); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetLeaderRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetLeaderRequestProcessor.java new file mode 100644 index 0000000..e7e1608 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetLeaderRequestProcessor.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.GetLeaderResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Process get leader request. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class GetLeaderRequestProcessor extends BaseCliRequestProcessor { + + public GetLeaderRequestProcessor(Executor executor) { + super(executor, GetLeaderResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final GetLeaderRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final GetLeaderRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final GetLeaderRequest request, + final RpcRequestClosure done) { + // ignore + return null; + } + + @Override + public Message processRequest(final GetLeaderRequest request, final RpcRequestClosure done) { + List nodes = new ArrayList<>(); + final String groupId = getGroupId(request); + if (request.hasPeerId()) { + final String peerIdStr = getPeerId(request); + final PeerId peer = new PeerId(); + if (peer.parse(peerIdStr)) { + final Status st = new Status(); + nodes.add(getNode(groupId, peer, st)); + if (!st.isOk()) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), st); + } + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerIdStr); + } + } else { + nodes = NodeManager.getInstance().getNodesByGroupId(groupId); + } + if (nodes == null || nodes.isEmpty()) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.ENOENT, "No nodes in group %s", groupId); + } + for (final Node node : nodes) { + final PeerId leader = node.getLeaderId(); + if (leader != null && !leader.isEmpty()) { + return GetLeaderResponse.newBuilder().setLeaderId(leader.toString()).build(); + } + } + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EAGAIN, "Unknown leader"); + } + + @Override + public String interest() { + return GetLeaderRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessor.java new file mode 100644 index 0000000..f7e9d07 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessor.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.google.protobuf.Message; + +/** + * Process get all peers of the replication group request. + * + * @author jiachun.fjc + */ +public class GetPeersRequestProcessor extends BaseCliRequestProcessor { + + public GetPeersRequestProcessor(final Executor executor) { + super(executor, GetPeersResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final GetPeersRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final GetPeersRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final GetPeersRequest request, + final RpcRequestClosure done) { + final List peers; + final List learners; + if (request.hasOnlyAlive() && request.getOnlyAlive()) { + peers = ctx.node.listAlivePeers(); + learners = ctx.node.listAliveLearners(); + } else { + peers = ctx.node.listPeers(); + learners = ctx.node.listLearners(); + } + final GetPeersResponse.Builder builder = GetPeersResponse.newBuilder(); + for (final PeerId peerId : peers) { + builder.addPeers(peerId.toString()); + } + for (final PeerId peerId : learners) { + builder.addLearners(peerId.toString()); + } + return builder.build(); + } + + @Override + public String interest() { + return GetPeersRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessor.java new file mode 100644 index 0000000..f01d871 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessor.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * RemoveLearners request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * + */ +public class RemoveLearnersRequestProcessor extends BaseCliRequestProcessor { + + public RemoveLearnersRequestProcessor(final Executor executor) { + super(executor, LearnersOpResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final RemoveLearnersRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final RemoveLearnersRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final RemoveLearnersRequest request, + final RpcRequestClosure done) { + final List oldLearners = ctx.node.listLearners(); + final List removeingLearners = new ArrayList<>(request.getLearnersCount()); + + for (final String peerStr : request.getLearnersList()) { + final PeerId peer = new PeerId(); + if (!peer.parse(peerStr)) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerStr); + } + removeingLearners.add(peer); + } + + LOG.info("Receive RemoveLearnersRequest to {} from {}, removing {}.", ctx.node.getNodeId(), + done.getRpcCtx().getRemoteAddress(), removeingLearners); + ctx.node.removeLearners(removeingLearners, status -> { + if (!status.isOk()) { + done.run(status); + } else { + final LearnersOpResponse.Builder rb = LearnersOpResponse.newBuilder(); + + for (final PeerId peer : oldLearners) { + rb.addOldLearners(peer.toString()); + if (!removeingLearners.contains(peer)) { + rb.addNewLearners(peer.toString()); + } + } + + done.sendResponse(rb.build()); + } + }); + + return null; + } + + @Override + public String interest() { + return RemoveLearnersRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessor.java new file mode 100644 index 0000000..37392f3 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessor.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest; +import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Remove peer request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class RemovePeerRequestProcessor extends BaseCliRequestProcessor { + + public RemovePeerRequestProcessor(Executor executor) { + super(executor, RemovePeerResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final RemovePeerRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final RemovePeerRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final RemovePeerRequest request, final RpcRequestClosure done) { + final List oldPeers = ctx.node.listPeers(); + final String removingPeerIdStr = request.getPeerId(); + final PeerId removingPeer = new PeerId(); + if (removingPeer.parse(removingPeerIdStr)) { + LOG.info("Receive RemovePeerRequest to {} from {}, removing {}", ctx.node.getNodeId(), done.getRpcCtx() + .getRemoteAddress(), removingPeerIdStr); + ctx.node.removePeer(removingPeer, status -> { + if (!status.isOk()) { + done.run(status); + } else { + final RemovePeerResponse.Builder rb = RemovePeerResponse.newBuilder(); + for (final PeerId oldPeer : oldPeers) { + rb.addOldPeers(oldPeer.toString()); + if (!oldPeer.equals(removingPeer)) { + rb.addNewPeers(oldPeer.toString()); + } + } + done.sendResponse(rb.build()); + } + }); + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", removingPeerIdStr); + } + + return null; + } + + @Override + public String interest() { + return RemovePeerRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessor.java new file mode 100644 index 0000000..69f40d2 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessor.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * ResetLearners request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * + */ +public class ResetLearnersRequestProcessor extends BaseCliRequestProcessor { + + public ResetLearnersRequestProcessor(final Executor executor) { + super(executor, LearnersOpResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final ResetLearnersRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final ResetLearnersRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final ResetLearnersRequest request, + final RpcRequestClosure done) { + final List oldLearners = ctx.node.listLearners(); + final List newLearners = new ArrayList<>(request.getLearnersCount()); + + for (final String peerStr : request.getLearnersList()) { + final PeerId peer = new PeerId(); + if (!peer.parse(peerStr)) { + return RpcFactoryHelper + .responseFactory() + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerStr); + } + newLearners.add(peer); + } + + LOG.info("Receive ResetLearnersRequest to {} from {}, resetting into {}.", ctx.node.getNodeId(), + done.getRpcCtx().getRemoteAddress(), newLearners); + ctx.node.resetLearners(newLearners, status -> { + if (!status.isOk()) { + done.run(status); + } else { + final LearnersOpResponse.Builder rb = LearnersOpResponse.newBuilder(); + + for (final PeerId peer : oldLearners) { + rb.addOldLearners(peer.toString()); + } + + for (final PeerId peer : newLearners) { + rb.addNewLearners(peer.toString()); + } + + done.sendResponse(rb.build()); + } + }); + + return null; + } + + @Override + public String interest() { + return ResetLearnersRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeerRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeerRequestProcessor.java new file mode 100644 index 0000000..26bbee1 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeerRequestProcessor.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Reset peer request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class ResetPeerRequestProcessor extends BaseCliRequestProcessor { + + public ResetPeerRequestProcessor(Executor executor) { + super(executor, RpcRequests.ErrorResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final ResetPeerRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final ResetPeerRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final ResetPeerRequest request, + final RpcRequestClosure done) { + final Configuration newConf = new Configuration(); + for (final String peerIdStr : request.getNewPeersList()) { + final PeerId peer = new PeerId(); + if (peer.parse(peerIdStr)) { + newConf.addPeer(peer); + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", peerIdStr); + } + } + LOG.info("Receive ResetPeerRequest to {} from {}, new conf is {}", ctx.node.getNodeId(), done.getRpcCtx() + .getRemoteAddress(), newConf); + final Status st = ctx.node.resetPeers(newConf); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), st); + } + + @Override + public String interest() { + return ResetPeerRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessor.java new file mode 100644 index 0000000..0c5357b --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessor.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.google.protobuf.Message; + +/** + * Snapshot request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class SnapshotRequestProcessor extends BaseCliRequestProcessor { + + public SnapshotRequestProcessor(Executor executor) { + super(executor, RpcRequests.ErrorResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final SnapshotRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final SnapshotRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final SnapshotRequest request, + final RpcRequestClosure done) { + LOG.info("Receive SnapshotRequest to {} from {}", ctx.node.getNodeId(), request.getPeerId()); + ctx.node.snapshot(done); + return null; + } + + @Override + public String interest() { + return SnapshotRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeaderRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeaderRequestProcessor.java new file mode 100644 index 0000000..dc0d616 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeaderRequestProcessor.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.cli; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Snapshot request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class TransferLeaderRequestProcessor extends BaseCliRequestProcessor { + + public TransferLeaderRequestProcessor(Executor executor) { + super(executor, RpcRequests.ErrorResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final TransferLeaderRequest request) { + return request.getLeaderId(); + } + + @Override + protected String getGroupId(final TransferLeaderRequest request) { + return request.getGroupId(); + } + + @Override + protected Message processRequest0(final CliRequestContext ctx, final TransferLeaderRequest request, + final RpcRequestClosure done) { + final PeerId peer = new PeerId(); + if (request.hasPeerId() && !peer.parse(request.getPeerId())) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peer id %s", request.getPeerId()); + } + LOG.info("Receive TransferLeaderRequest to {} from {}, newLeader will be {}.", ctx.node.getNodeId(), done + .getRpcCtx().getRemoteAddress(), peer); + final Status st = ctx.node.transferLeadershipTo(peer); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), st); + } + + @Override + public String interest() { + return TransferLeaderRequest.class.getName(); + } + +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessor.java new file mode 100644 index 0000000..2e79fc0 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessor.java @@ -0,0 +1,517 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.HashMap; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import com.alipay.sofa.jraft.JRaftUtils; +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.rpc.Connection; +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcContext; +import com.alipay.sofa.jraft.rpc.RpcProcessor; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader; +import com.alipay.sofa.jraft.rpc.impl.ConnectionClosedEventListener; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.alipay.sofa.jraft.util.Utils; +import com.alipay.sofa.jraft.util.concurrent.ConcurrentHashSet; +import com.alipay.sofa.jraft.util.concurrent.MpscSingleThreadExecutor; +import com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutor; +import com.google.protobuf.Message; + +/** + * Append entries request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 3:00:13 PM + */ +public class AppendEntriesRequestProcessor extends NodeRequestProcessor implements + ConnectionClosedEventListener { + + static final String PAIR_ATTR = "jraft-peer-pairs"; + + /** + * Peer executor selector. + * + * @author dennis + */ + final class PeerExecutorSelector implements RpcProcessor.ExecutorSelector { + + PeerExecutorSelector() { + super(); + } + + @Override + public Executor select(final String reqClass, final Object reqHeader) { + final AppendEntriesRequestHeader header = (AppendEntriesRequestHeader) reqHeader; + final String groupId = header.getGroupId(); + final String peerId = header.getPeerId(); + final String serverId = header.getServerId(); + + final PeerId peer = new PeerId(); + + if (!peer.parse(peerId)) { + return executor(); + } + + final Node node = NodeManager.getInstance().get(groupId, peer); + + if (node == null || !node.getRaftOptions().isReplicatorPipeline()) { + return executor(); + } + + // The node enable pipeline, we should ensure bolt support it. + RpcFactoryHelper.rpcFactory().ensurePipeline(); + + final PeerRequestContext ctx = getOrCreatePeerRequestContext(groupId, pairOf(peerId, serverId), null); + + return ctx.executor; + } + } + + /** + * RpcRequestClosure that will send responses in pipeline mode. + * + * @author dennis + */ + class SequenceRpcRequestClosure extends RpcRequestClosure { + + private final int reqSequence; + private final String groupId; + private final PeerPair pair; + private final boolean isHeartbeat; + + public SequenceRpcRequestClosure(final RpcRequestClosure parent, final Message defaultResp, + final String groupId, final PeerPair pair, final int sequence, + final boolean isHeartbeat) { + super(parent.getRpcCtx(), defaultResp); + this.reqSequence = sequence; + this.groupId = groupId; + this.pair = pair; + this.isHeartbeat = isHeartbeat; + } + + @Override + public void sendResponse(final Message msg) { + if (this.isHeartbeat) { + super.sendResponse(msg); + } else { + sendSequenceResponse(this.groupId, this.pair, this.reqSequence, getRpcCtx(), msg); + } + } + } + + /** + * Response message wrapper with a request sequence number and asyncContext.done + * + * @author dennis + */ + static class SequenceMessage implements Comparable { + public final Message msg; + private final int sequence; + private final RpcContext rpcCtx; + + public SequenceMessage(final RpcContext rpcCtx, final Message msg, final int sequence) { + super(); + this.rpcCtx = rpcCtx; + this.msg = msg; + this.sequence = sequence; + } + + /** + * Send the response. + */ + void sendResponse() { + this.rpcCtx.sendResponse(this.msg); + } + + /** + * Order by sequence number + */ + @Override + public int compareTo(final SequenceMessage o) { + return Integer.compare(this.sequence, o.sequence); + } + } + + // constant pool for peer pair + private final Map> pairConstants = new HashMap<>(); + + PeerPair pairOf(final String peerId, final String serverId) { + synchronized (this.pairConstants) { + Map pairs = this.pairConstants.computeIfAbsent(peerId, k -> new HashMap<>()); + + PeerPair pair = pairs.computeIfAbsent(serverId, k -> new PeerPair(peerId, serverId)); + return pair; + } + } + + /** + * A peer pair + * @author boyan(boyan@antfin.com) + * + */ + static class PeerPair { + // peer in local node + final String local; + // peer in remote node + final String remote; + + PeerPair(final String local, final String remote) { + super(); + this.local = local; + this.remote = remote; + } + + @Override + public String toString() { + return "PeerPair[" + this.local + " -> " + this.remote + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((this.local == null) ? 0 : this.local.hashCode()); + result = prime * result + ((this.remote == null) ? 0 : this.remote.hashCode()); + return result; + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + PeerPair other = (PeerPair) obj; + if (this.local == null) { + if (other.local != null) { + return false; + } + } else if (!this.local.equals(other.local)) { + return false; + } + if (this.remote == null) { + if (other.remote != null) { + return false; + } + } else if (!this.remote.equals(other.remote)) { + return false; + } + return true; + } + } + + static class PeerRequestContext { + + private final String groupId; + private final PeerPair pair; + + // Executor to run the requests + private SingleThreadExecutor executor; + // The request sequence; + private int sequence; + // The required sequence to be sent. + private int nextRequiredSequence; + // The response queue,it's not thread-safe and protected by it self object monitor. + private final PriorityQueue responseQueue; + + private final int maxPendingResponses; + + public PeerRequestContext(final String groupId, final PeerPair pair, final int maxPendingResponses) { + super(); + this.pair = pair; + this.groupId = groupId; + this.executor = new MpscSingleThreadExecutor(Utils.MAX_APPEND_ENTRIES_TASKS_PER_THREAD, + JRaftUtils.createThreadFactory(groupId + "/" + pair + "-AppendEntriesThread")); + + this.sequence = 0; + this.nextRequiredSequence = 0; + this.maxPendingResponses = maxPendingResponses; + this.responseQueue = new PriorityQueue<>(50); + } + + boolean hasTooManyPendingResponses() { + return this.responseQueue.size() > this.maxPendingResponses; + } + + int getAndIncrementSequence() { + final int prev = this.sequence; + this.sequence++; + if (this.sequence < 0) { + this.sequence = 0; + } + return prev; + } + + synchronized void destroy() { + if (this.executor != null) { + LOG.info("Destroyed peer request context for {}/{}", this.groupId, this.pair); + this.executor.shutdownGracefully(); + this.executor = null; + } + } + + int getNextRequiredSequence() { + return this.nextRequiredSequence; + } + + int getAndIncrementNextRequiredSequence() { + final int prev = this.nextRequiredSequence; + this.nextRequiredSequence++; + if (this.nextRequiredSequence < 0) { + this.nextRequiredSequence = 0; + } + return prev; + } + } + + PeerRequestContext getPeerRequestContext(final String groupId, final PeerPair pair) { + ConcurrentMap groupContexts = this.peerRequestContexts.get(groupId); + + if (groupContexts == null) { + return null; + } + return groupContexts.get(pair); + } + + /** + * Send request in pipeline mode. + */ + void sendSequenceResponse(final String groupId, final PeerPair pair, final int seq, final RpcContext rpcCtx, + final Message msg) { + final PeerRequestContext ctx = getPeerRequestContext(groupId, pair); + if (ctx == null) { + // the context was destroyed, so the response can be ignored. + return; + } + final PriorityQueue respQueue = ctx.responseQueue; + assert (respQueue != null); + + synchronized (Utils.withLockObject(respQueue)) { + respQueue.add(new SequenceMessage(rpcCtx, msg, seq)); + + if (!ctx.hasTooManyPendingResponses()) { + while (!respQueue.isEmpty()) { + final SequenceMessage queuedPipelinedResponse = respQueue.peek(); + + if (queuedPipelinedResponse.sequence != ctx.getNextRequiredSequence()) { + // sequence mismatch, waiting for next response. + break; + } + respQueue.remove(); + try { + queuedPipelinedResponse.sendResponse(); + } finally { + ctx.getAndIncrementNextRequiredSequence(); + } + } + } else { + final Connection connection = rpcCtx.getConnection(); + LOG.warn("Closed connection to peer {}/{}, because of too many pending responses, queued={}, max={}", + ctx.groupId, pair, respQueue.size(), ctx.maxPendingResponses); + connection.close(); + // Close the connection if there are too many pending responses in queue. + removePeerRequestContext(groupId, pair); + } + } + } + + @SuppressWarnings("unchecked") + PeerRequestContext getOrCreatePeerRequestContext(final String groupId, final PeerPair pair, final Connection conn) { + ConcurrentMap groupContexts = this.peerRequestContexts.get(groupId); + if (groupContexts == null) { + groupContexts = new ConcurrentHashMap<>(); + final ConcurrentMap existsCtxs = this.peerRequestContexts.putIfAbsent( + groupId, groupContexts); + if (existsCtxs != null) { + groupContexts = existsCtxs; + } + } + + PeerRequestContext peerCtx = groupContexts.get(pair); + if (peerCtx == null) { + synchronized (Utils.withLockObject(groupContexts)) { + peerCtx = groupContexts.get(pair); + // double check in lock + if (peerCtx == null) { + // only one thread to process append entries for every jraft node + final PeerId peer = new PeerId(); + final boolean parsed = peer.parse(pair.local); + assert (parsed); + final Node node = NodeManager.getInstance().get(groupId, peer); + assert (node != null); + peerCtx = new PeerRequestContext(groupId, pair, node.getRaftOptions() + .getMaxReplicatorInflightMsgs()); + groupContexts.put(pair, peerCtx); + } + } + } + + // Add the pair to connection attribute metadata. + if (conn != null) { + Set pairs; + if ((pairs = (Set) conn.getAttribute(PAIR_ATTR)) == null) { + pairs = new ConcurrentHashSet<>(); + Set existsPairs = (Set) conn.setAttributeIfAbsent(PAIR_ATTR, pairs); + if (existsPairs != null) { + pairs = existsPairs; + } + } + + pairs.add(pair); + } + return peerCtx; + } + + void removePeerRequestContext(final String groupId, final PeerPair pair) { + final ConcurrentMap groupContexts = this.peerRequestContexts.get(groupId); + if (groupContexts == null) { + return; + } + synchronized (Utils.withLockObject(groupContexts)) { + final PeerRequestContext ctx = groupContexts.remove(pair); + if (ctx != null) { + ctx.destroy(); + } + } + } + + /** + * RAFT group peer request contexts. + */ + private final ConcurrentMap> peerRequestContexts = new ConcurrentHashMap<>(); + + /** + * The executor selector to select executor for processing request. + */ + private final ExecutorSelector executorSelector; + + public AppendEntriesRequestProcessor(final Executor executor) { + super(executor, RpcRequests.AppendEntriesResponse.getDefaultInstance()); + this.executorSelector = new PeerExecutorSelector(); + } + + @Override + protected String getPeerId(final AppendEntriesRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final AppendEntriesRequest request) { + return request.getGroupId(); + } + + private int getAndIncrementSequence(final String groupId, final PeerPair pair, final Connection conn) { + return getOrCreatePeerRequestContext(groupId, pair, conn).getAndIncrementSequence(); + } + + private boolean isHeartbeatRequest(final AppendEntriesRequest request) { + // No entries and no data means a true heartbeat request. + // TODO(boyan) refactor, adds a new flag field? + return request.getEntriesCount() == 0 && !request.hasData(); + } + + @Override + public Message processRequest0(final RaftServerService service, final AppendEntriesRequest request, + final RpcRequestClosure done) { + + final Node node = (Node) service; + + if (node.getRaftOptions().isReplicatorPipeline()) { + final String groupId = request.getGroupId(); + final PeerPair pair = pairOf(request.getPeerId(), request.getServerId()); + + boolean isHeartbeat = isHeartbeatRequest(request); + int reqSequence = -1; + if (!isHeartbeat) { + reqSequence = getAndIncrementSequence(groupId, pair, done.getRpcCtx().getConnection()); + } + final Message response = service.handleAppendEntriesRequest(request, new SequenceRpcRequestClosure(done, + defaultResp(), groupId, pair, reqSequence, isHeartbeat)); + if (response != null) { + if (isHeartbeat) { + done.getRpcCtx().sendResponse(response); + } else { + sendSequenceResponse(groupId, pair, reqSequence, done.getRpcCtx(), response); + } + } + return null; + } else { + return service.handleAppendEntriesRequest(request, done); + } + } + + @Override + public String interest() { + return AppendEntriesRequest.class.getName(); + } + + @Override + public ExecutorSelector executorSelector() { + return this.executorSelector; + } + + // TODO called when shutdown service. + public void destroy() { + for (final ConcurrentMap map : this.peerRequestContexts.values()) { + for (final PeerRequestContext ctx : map.values()) { + ctx.destroy(); + } + } + this.peerRequestContexts.clear(); + } + + @SuppressWarnings("unchecked") + @Override + public void onClosed(final String remoteAddress, final Connection conn) { + final Set pairs = (Set) conn.getAttribute(PAIR_ATTR); + if (pairs != null && !pairs.isEmpty()) { + // Clear request contexts when connection disconnected. + for (final Map.Entry> entry : this.peerRequestContexts + .entrySet()) { + final ConcurrentMap groupCtxs = entry.getValue(); + synchronized (Utils.withLockObject(groupCtxs)) { + for (PeerPair pair : pairs) { + final PeerRequestContext ctx = groupCtxs.remove(pair); + if (ctx != null) { + ctx.destroy(); + } + } + } + } + } else { + LOG.info("Connection disconnected: {}", remoteAddress); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ClientServiceConnectionEventProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ClientServiceConnectionEventProcessor.java new file mode 100644 index 0000000..d83a60c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ClientServiceConnectionEventProcessor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.remoting.Connection; +import com.alipay.remoting.ConnectionEventProcessor; +import com.alipay.remoting.ConnectionEventType; +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.entity.PeerId; + +/** + * Client RPC service connection event processor for {@link ConnectionEventType#CONNECT} + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-12 10:21:22 AM + */ +public class ClientServiceConnectionEventProcessor implements ConnectionEventProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(ClientServiceConnectionEventProcessor.class); + + private final ReplicatorGroup rgGroup; + + public ClientServiceConnectionEventProcessor(ReplicatorGroup rgGroup) { + super(); + this.rgGroup = rgGroup; + } + + @Override + public void onEvent(final String remoteAddr, final Connection conn) { + final PeerId peer = new PeerId(); + if (peer.parse(remoteAddr)) { + LOG.info("Peer {} is connected", peer); + this.rgGroup.checkReplicator(peer, true); + } else { + LOG.error("Fail to parse peer: {}", remoteAddr); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientService.java new file mode 100644 index 0000000..b1b3995 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientService.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executor; +import java.util.concurrent.Future; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.ReplicatorGroup; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RemotingException; +import com.alipay.sofa.jraft.option.NodeOptions; +import com.alipay.sofa.jraft.option.RpcOptions; +import com.alipay.sofa.jraft.rpc.InvokeContext; +import com.alipay.sofa.jraft.rpc.RaftClientService; +import com.alipay.sofa.jraft.rpc.RpcClient; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteResponse; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowResponse; +import com.alipay.sofa.jraft.rpc.RpcResponseClosure; +import com.alipay.sofa.jraft.rpc.impl.AbstractClientService; +import com.alipay.sofa.jraft.rpc.impl.FutureImpl; +import com.alipay.sofa.jraft.util.Endpoint; +import com.alipay.sofa.jraft.util.Utils; +import com.alipay.sofa.jraft.util.concurrent.DefaultFixedThreadsExecutorGroupFactory; +import com.alipay.sofa.jraft.util.concurrent.FixedThreadsExecutorGroup; +import com.google.protobuf.Message; + +/** + * Raft rpc service based bolt. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class DefaultRaftClientService extends AbstractClientService implements RaftClientService { + + private static final FixedThreadsExecutorGroup APPEND_ENTRIES_EXECUTORS = DefaultFixedThreadsExecutorGroupFactory.INSTANCE + .newExecutorGroup( + Utils.APPEND_ENTRIES_THREADS_SEND, + "Append-Entries-Thread-Send", + Utils.MAX_APPEND_ENTRIES_TASKS_PER_THREAD, + true); + + private final ConcurrentMap appendEntriesExecutorMap = new ConcurrentHashMap<>(); + + // cached node options + private NodeOptions nodeOptions; + private final ReplicatorGroup rgGroup; + + @Override + protected void configRpcClient(final RpcClient rpcClient) { + rpcClient.registerConnectEventListener(this.rgGroup); + } + + public DefaultRaftClientService(final ReplicatorGroup rgGroup) { + this.rgGroup = rgGroup; + } + + @Override + public synchronized boolean init(final RpcOptions rpcOptions) { + final boolean ret = super.init(rpcOptions); + if (ret) { + this.nodeOptions = (NodeOptions) rpcOptions; + } + return ret; + } + + @Override + public Future preVote(final Endpoint endpoint, final RequestVoteRequest request, + final RpcResponseClosure done) { + if (!checkConnection(endpoint, true)) { + return onConnectionFail(endpoint, request, done, this.rpcExecutor); + } + + return invokeWithDone(endpoint, request, done, this.nodeOptions.getElectionTimeoutMs()); + } + + @Override + public Future requestVote(final Endpoint endpoint, final RequestVoteRequest request, + final RpcResponseClosure done) { + if (!checkConnection(endpoint, true)) { + return onConnectionFail(endpoint, request, done, this.rpcExecutor); + } + + return invokeWithDone(endpoint, request, done, this.nodeOptions.getElectionTimeoutMs()); + } + + @Override + public Future appendEntries(final Endpoint endpoint, final AppendEntriesRequest request, + final int timeoutMs, final RpcResponseClosure done) { + final Executor executor = this.appendEntriesExecutorMap.computeIfAbsent(endpoint, k -> APPEND_ENTRIES_EXECUTORS.next()); + + if (!checkConnection(endpoint, true)) { + return onConnectionFail(endpoint, request, done, executor); + } + + return invokeWithDone(endpoint, request, done, timeoutMs, executor); + } + + @Override + public Future getFile(final Endpoint endpoint, final GetFileRequest request, final int timeoutMs, + final RpcResponseClosure done) { + // open checksum + final InvokeContext ctx = new InvokeContext(); + ctx.put(InvokeContext.CRC_SWITCH, true); + return invokeWithDone(endpoint, request, ctx, done, timeoutMs); + } + + @Override + public Future installSnapshot(final Endpoint endpoint, final InstallSnapshotRequest request, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, this.rpcOptions.getRpcInstallSnapshotTimeout()); + } + + @Override + public Future timeoutNow(final Endpoint endpoint, final TimeoutNowRequest request, final int timeoutMs, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, timeoutMs); + } + + @Override + public Future readIndex(final Endpoint endpoint, final ReadIndexRequest request, final int timeoutMs, + final RpcResponseClosure done) { + return invokeWithDone(endpoint, request, done, timeoutMs); + } + + // fail-fast when no connection + private Future onConnectionFail(final Endpoint endpoint, final Message request, Closure done, final Executor executor) { + final FutureImpl future = new FutureImpl<>(); + executor.execute(() -> { + final String fmt = "Check connection[%s] fail and try to create new one"; + if (done != null) { + try { + done.run(new Status(RaftError.EINTERNAL, fmt, endpoint)); + } catch (final Throwable t) { + LOG.error("Fail to run RpcResponseClosure, the request is {}.", request, t); + } + } + if (!future.isDone()) { + future.failure(new RemotingException(String.format(fmt, endpoint))); + } + }); + return future; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/GetFileRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/GetFileRequestProcessor.java new file mode 100644 index 0000000..9e0a4c4 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/GetFileRequestProcessor.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest; +import com.alipay.sofa.jraft.storage.FileService; +import com.google.protobuf.Message; + +/** + * Get file request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 3:01:25 PM + */ +public class GetFileRequestProcessor extends RpcRequestProcessor { + + public GetFileRequestProcessor(Executor executor) { + super(executor, RpcRequests.GetFileResponse.getDefaultInstance()); + } + + @Override + public Message processRequest(final GetFileRequest request, final RpcRequestClosure done) { + return FileService.getInstance().handleGetFile(request, done); + } + + @Override + public String interest() { + return GetFileRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessor.java new file mode 100644 index 0000000..0488d24 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessor.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.google.protobuf.Message; + +/** + * Handle install snapshot request. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 6:09:34 PM + */ +public class InstallSnapshotRequestProcessor extends NodeRequestProcessor { + + public InstallSnapshotRequestProcessor(Executor executor) { + super(executor, RpcRequests.InstallSnapshotResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final InstallSnapshotRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final InstallSnapshotRequest request) { + return request.getGroupId(); + } + + @Override + public Message processRequest0(final RaftServerService service, final InstallSnapshotRequest request, + final RpcRequestClosure done) { + return service.handleInstallSnapshot(request, done); + } + + @Override + public String interest() { + return InstallSnapshotRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessor.java new file mode 100644 index 0000000..f4ced1a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessor.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.Node; +import com.alipay.sofa.jraft.NodeManager; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequestProcessor; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.google.protobuf.Message; + +/** + * Node handle requests processor template. + * + * @param Message + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public abstract class NodeRequestProcessor extends RpcRequestProcessor { + + public NodeRequestProcessor(Executor executor, Message defaultResp) { + super(executor, defaultResp); + } + + protected abstract Message processRequest0(final RaftServerService serviceService, final T request, + final RpcRequestClosure done); + + protected abstract String getPeerId(final T request); + + protected abstract String getGroupId(final T request); + + @Override + public Message processRequest(final T request, final RpcRequestClosure done) { + final PeerId peer = new PeerId(); + final String peerIdStr = getPeerId(request); + if (peer.parse(peerIdStr)) { + final String groupId = getGroupId(request); + final Node node = NodeManager.getInstance().get(groupId, peer); + if (node != null) { + return processRequest0((RaftServerService) node, request, done); + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.ENOENT, "Peer id not found: %s, group: %s", peerIdStr, + groupId); + } + } else { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(defaultResp(), RaftError.EINVAL, "Fail to parse peerId: %s", peerIdStr); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessor.java new file mode 100644 index 0000000..c64ceca --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessor.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest; +import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter; +import com.google.protobuf.Message; + +/** + * Handle read index request. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class ReadIndexRequestProcessor extends NodeRequestProcessor { + + public ReadIndexRequestProcessor(Executor executor) { + super(executor, RpcRequests.ReadIndexResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final ReadIndexRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final ReadIndexRequest request) { + return request.getGroupId(); + } + + @Override + public Message processRequest0(final RaftServerService service, final ReadIndexRequest request, + final RpcRequestClosure done) { + service.handleReadIndexRequest(request, new RpcResponseClosureAdapter() { + + @Override + public void run(final Status status) { + if (getResponse() != null) { + done.sendResponse(getResponse()); + } else { + done.run(status); + } + } + + }); + return null; + } + + @Override + public String interest() { + return ReadIndexRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessor.java new file mode 100644 index 0000000..ea287e9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessor.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest; +import com.google.protobuf.Message; + +/** + * Handle PreVote and RequestVote requests. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 6:11:09 PM + */ +public class RequestVoteRequestProcessor extends NodeRequestProcessor { + + public RequestVoteRequestProcessor(Executor executor) { + super(executor, RpcRequests.RequestVoteResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final RequestVoteRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final RequestVoteRequest request) { + return request.getGroupId(); + } + + @Override + public Message processRequest0(final RaftServerService service, final RequestVoteRequest request, + final RpcRequestClosure done) { + if (request.getPreVote()) { + return service.handlePreVoteRequest(request); + } else { + return service.handleRequestVoteRequest(request); + } + } + + @Override + public String interest() { + return RequestVoteRequest.class.getName(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessor.java new file mode 100644 index 0000000..5fd9c19 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessor.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.rpc.impl.core; + +import java.util.concurrent.Executor; + +import com.alipay.sofa.jraft.rpc.RaftServerService; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests; +import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest; +import com.google.protobuf.Message; + +/** + * TimeoutNow request processor. + * + * @author boyan (boyan@alibaba-inc.com) + * @author jiachun.fjc + */ +public class TimeoutNowRequestProcessor extends NodeRequestProcessor { + + public TimeoutNowRequestProcessor(Executor executor) { + super(executor, RpcRequests.TimeoutNowResponse.getDefaultInstance()); + } + + @Override + protected String getPeerId(final TimeoutNowRequest request) { + return request.getPeerId(); + } + + @Override + protected String getGroupId(final TimeoutNowRequest request) { + return request.getGroupId(); + } + + @Override + public Message processRequest0(final RaftServerService service, final TimeoutNowRequest request, + final RpcRequestClosure done) { + return service.handleTimeoutNowRequest(request, done); + } + + @Override + public String interest() { + return TimeoutNowRequest.class.getName(); + } +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/FileService.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/FileService.java new file mode 100644 index 0000000..96bb0dd --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/FileService.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; + +import io.netty.util.internal.ThreadLocalRandom; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RetryAgainException; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse; +import com.alipay.sofa.jraft.storage.io.FileReader; +import com.alipay.sofa.jraft.util.ByteBufferCollector; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.RpcFactoryHelper; +import com.alipay.sofa.jraft.util.Utils; +import com.google.protobuf.ByteString; +import com.google.protobuf.Message; +import com.google.protobuf.ZeroByteStringHelper; + +/** + * File reader service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-30 10:23:13 AM + */ +public final class FileService { + + private static final Logger LOG = LoggerFactory.getLogger(FileService.class); + + private static final FileService INSTANCE = new FileService(); + + private final ConcurrentMap fileReaderMap = new ConcurrentHashMap<>(); + private final AtomicLong nextId = new AtomicLong(); + + /** + * Retrieve the singleton instance of FileService. + * + * @return a fileService instance + */ + public static FileService getInstance() { + return INSTANCE; + } + + @OnlyForTest + void clear() { + this.fileReaderMap.clear(); + } + + private FileService() { + final long processId = Utils.getProcessId(ThreadLocalRandom.current().nextLong(10000, Integer.MAX_VALUE)); + final long initialValue = Math.abs(processId << 45 | System.nanoTime() << 17 >> 17); + this.nextId.set(initialValue); + LOG.info("Initial file reader id in FileService is {}", initialValue); + } + + /** + * Handle GetFileRequest, run the response or set the response with done. + */ + public Message handleGetFile(final GetFileRequest request, final RpcRequestClosure done) { + if (request.getCount() <= 0 || request.getOffset() < 0) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(GetFileResponse.getDefaultInstance(), RaftError.EREQUEST, "Invalid request: %s", request); + } + final FileReader reader = this.fileReaderMap.get(request.getReaderId()); + if (reader == null) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(GetFileResponse.getDefaultInstance(), RaftError.ENOENT, "Fail to find reader=%d", + request.getReaderId()); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("GetFile from {} path={} filename={} offset={} count={}", done.getRpcCtx().getRemoteAddress(), + reader.getPath(), request.getFilename(), request.getOffset(), request.getCount()); + } + + final ByteBufferCollector dataBuffer = ByteBufferCollector.allocate(); + final GetFileResponse.Builder responseBuilder = GetFileResponse.newBuilder(); + try { + final int read = reader + .readFile(dataBuffer, request.getFilename(), request.getOffset(), request.getCount()); + responseBuilder.setReadSize(read); + responseBuilder.setEof(read == FileReader.EOF); + final ByteBuffer buf = dataBuffer.getBuffer(); + buf.flip(); + if (!buf.hasRemaining()) { + // skip empty data + responseBuilder.setData(ByteString.EMPTY); + } else { + // TODO check hole + responseBuilder.setData(ZeroByteStringHelper.wrap(buf)); + } + return responseBuilder.build(); + } catch (final RetryAgainException e) { + return RpcFactoryHelper // + .responseFactory() // + .newResponse(GetFileResponse.getDefaultInstance(), RaftError.EAGAIN, + "Fail to read from path=%s filename=%s with error: %s", reader.getPath(), request.getFilename(), + e.getMessage()); + } catch (final IOException e) { + LOG.error("Fail to read file path={} filename={}", reader.getPath(), request.getFilename(), e); + return RpcFactoryHelper // + .responseFactory() // + .newResponse(GetFileResponse.getDefaultInstance(), RaftError.EIO, + "Fail to read from path=%s filename=%s", reader.getPath(), request.getFilename()); + } + } + + /** + * Adds a file reader and return it's generated readerId. + */ + public long addReader(final FileReader reader) { + final long readerId = this.nextId.getAndIncrement(); + if (this.fileReaderMap.putIfAbsent(readerId, reader) == null) { + return readerId; + } else { + return -1L; + } + } + + /** + * Remove the reader by readerId. + */ + public boolean removeReader(final long readerId) { + return this.fileReaderMap.remove(readerId) != null; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogManager.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogManager.java new file mode 100644 index 0000000..9a70f84 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogManager.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import java.util.List; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta; +import com.alipay.sofa.jraft.option.LogManagerOptions; +import com.alipay.sofa.jraft.util.Describer; + +/** + * Log manager. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 3:02:42 PM + */ +public interface LogManager extends Lifecycle, Describer { + + /** + * Closure to to run in stable state. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 4:35:29 PM + */ + abstract class StableClosure implements Closure { + + protected long firstLogIndex = 0; + protected List entries; + protected int nEntries; + + public StableClosure() { + // NO-OP + } + + public long getFirstLogIndex() { + return this.firstLogIndex; + } + + public void setFirstLogIndex(final long firstLogIndex) { + this.firstLogIndex = firstLogIndex; + } + + public List getEntries() { + return this.entries; + } + + public void setEntries(final List entries) { + this.entries = entries; + if (entries != null) { + this.nEntries = entries.size(); + } else { + this.nEntries = 0; + } + } + + public StableClosure(final List entries) { + super(); + setEntries(entries); + } + + } + + /** + * Listen on last log index change event, but it's not reliable, + * the user should not count on this listener to receive all changed events. + * + * @author dennis + */ + interface LastLogIndexListener { + + /** + * Called when last log index is changed. + * + * @param lastLogIndex last log index + */ + void onLastLogIndexChanged(final long lastLogIndex); + } + + /** + * Adds a last log index listener + */ + void addLastLogIndexListener(final LastLogIndexListener listener); + + /** + * Remove the last log index listener. + */ + void removeLastLogIndexListener(final LastLogIndexListener listener); + + /** + * Wait the log manager to be shut down. + * + * @throws InterruptedException if the current thread is interrupted + * while waiting + */ + void join() throws InterruptedException; + + /** + * Given specified requiredCapacity determines if that amount of space + * is available to append these entries. Returns true when available. + * @param requiredCapacity + * @return Returns true when available. + */ + boolean hasAvailableCapacityToAppendEntries(final int requiredCapacity); + + /** + * Append log entry vector and wait until it's stable (NOT COMMITTED!) + * + * @param entries log entries + * @param done callback + */ + void appendEntries(final List entries, StableClosure done); + + /** + * Notify the log manager about the latest snapshot, which indicates the + * logs which can be safely truncated. + * + * @param meta snapshot metadata + */ + void setSnapshot(final SnapshotMeta meta); + + /** + * We don't delete all the logs before last snapshot to avoid installing + * snapshot on slow replica. Call this method to drop all the logs before + * last snapshot immediately. + */ + void clearBufferedLogs(); + + /** + * Get the log entry at index. + * + * @param index the index of log entry + * @return the log entry with {@code index} + */ + LogEntry getEntry(final long index); + + /** + * Get the log term at index. + * + * @param index the index of log entry + * @return the term of log entry + */ + long getTerm(final long index); + + /** + * Get the first log index of log + */ + long getFirstLogIndex(); + + /** + * Get the last log index of log + */ + long getLastLogIndex(); + + /** + * Get the last log index of log + * + * @param isFlush whether to flush from disk. + */ + long getLastLogIndex(final boolean isFlush); + + /** + * Return the id the last log. + * + * @param isFlush whether to flush all pending task. + */ + LogId getLastLogId(final boolean isFlush); + + /** + * Get the configuration at index. + */ + ConfigurationEntry getConfiguration(final long index); + + /** + * Check if |current| should be updated to the latest configuration + * Returns the latest configuration, otherwise null. + */ + ConfigurationEntry checkAndSetConfiguration(final ConfigurationEntry current); + + /** + * New log notifier callback. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 4:40:04 PM + */ + interface NewLogCallback { + + /** + * Called while new log come in. + * + * @param arg the waiter pass-in argument + * @param errorCode error code + */ + boolean onNewLog(final Object arg, final int errorCode); + } + + /** + * Wait until there are more logs since |last_log_index| and |on_new_log| + * would be called after there are new logs or error occurs, return the waiter id. + * + * @param expectedLastLogIndex expected last index of log + * @param cb callback + * @param arg the waiter pass-in argument + */ + long wait(final long expectedLastLogIndex, final NewLogCallback cb, final Object arg); + + /** + * Remove a waiter. + * + * @param id waiter id + * @return true on success + */ + boolean removeWaiter(final long id); + + /** + * Set the applied id, indicating that the log before applied_id (included) + * can be dropped from memory logs. + */ + void setAppliedId(final LogId appliedId); + + /** + * Check log consistency, returns the status + * @return status + */ + Status checkConsistency(); + +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogStorage.java new file mode 100644 index 0000000..3adef4a --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/LogStorage.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import java.util.List; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.option.LogStorageOptions; + +/** + * Log entry storage service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:43:54 PM + */ +public interface LogStorage extends Lifecycle, Storage { + + /** + * Returns first log index in log. + */ + long getFirstLogIndex(); + + /** + * Returns last log index in log. + */ + long getLastLogIndex(); + + /** + * Get logEntry by index. + */ + LogEntry getEntry(final long index); + + /** + * Get logEntry's term by index. This method is deprecated, you should use {@link #getEntry(long)} to get the log id's term. + * @deprecated + */ + @Deprecated + long getTerm(final long index); + + /** + * Append entries to log. + */ + boolean appendEntry(final LogEntry entry); + + /** + * Append entries to log, return append success number. + */ + int appendEntries(final List entries); + + /** + * Delete logs from storage's head, [first_log_index, first_index_kept) will + * be discarded. + */ + boolean truncatePrefix(final long firstIndexKept); + + /** + * Delete uncommitted logs from storage's tail, (last_index_kept, last_log_index] + * will be discarded. + */ + boolean truncateSuffix(final long lastIndexKept); + + /** + * Drop all the existing logs and reset next log index to |next_log_index|. + * This function is called after installing snapshot from leader. + */ + boolean reset(final long nextLogIndex); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/RaftMetaStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/RaftMetaStorage.java new file mode 100644 index 0000000..f6c396b --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/RaftMetaStorage.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.option.RaftMetaStorageOptions; + +/** + * Raft metadata storage service. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:54:21 PM + */ +public interface RaftMetaStorage extends Lifecycle, Storage { + + /** + * Set current term. + */ + boolean setTerm(final long term); + + /** + * Get current term. + */ + long getTerm(); + + /** + * Set voted for information. + */ + boolean setVotedFor(final PeerId peerId); + + /** + * Get voted for information. + */ + PeerId getVotedFor(); + + /** + * Set term and voted for information. + */ + boolean setTermAndVotedFor(final long term, final PeerId peerId); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotExecutor.java new file mode 100644 index 0000000..0cb5026 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotExecutor.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import com.alipay.sofa.jraft.Closure; +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.option.SnapshotExecutorOptions; +import com.alipay.sofa.jraft.rpc.RpcRequestClosure; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest; +import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse; +import com.alipay.sofa.jraft.util.Describer; + +/** + * Executing Snapshot related stuff. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-22 2:27:02 PM + */ +public interface SnapshotExecutor extends Lifecycle, Describer { + + /** + * Return the owner NodeImpl + */ + NodeImpl getNode(); + + /** + * Start to snapshot StateMachine, and |done| is called after the + * execution finishes or fails. + * + * @param done snapshot callback + */ + void doSnapshot(final Closure done); + + /** + * Install snapshot according to the very RPC from leader + * After the installing succeeds (StateMachine is reset with the snapshot) + * or fails, done will be called to respond + * Errors: + * - Term mismatches: which happens interrupt_downloading_snapshot was + * called before install_snapshot, indicating that this RPC was issued by + * the old leader. + * - Interrupted: happens when interrupt_downloading_snapshot is called or + * a new RPC with the same or newer snapshot arrives + * - Busy: the state machine is saving or loading snapshot + */ + void installSnapshot(final InstallSnapshotRequest request, final InstallSnapshotResponse.Builder response, + final RpcRequestClosure done); + + /** + * Interrupt the downloading if possible. + * This is called when the term of node increased to |new_term|, which + * happens when receiving RPC from new peer. In this case, it's hard to + * determine whether to keep downloading snapshot as the new leader + * possibly contains the missing logs and is going to send AppendEntries. To + * make things simplicity and leader changing during snapshot installing is + * very rare. So we interrupt snapshot downloading when leader changes, and + * let the new leader decide whether to install a new snapshot or continue + * appending log entries. + * + * NOTE: we can't interrupt the snapshot installing which has finished + * downloading and is reseting the State Machine. + * + * @param newTerm new term num + */ + void interruptDownloadingSnapshots(final long newTerm); + + /** + * Returns true if this is currently installing a snapshot, either + * downloading or loading. + */ + boolean isInstallingSnapshot(); + + /** + * Returns the backing snapshot storage + */ + SnapshotStorage getSnapshotStorage(); + + /** + * Block the current thread until all the running job finishes (including failure) + */ + void join() throws InterruptedException; +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotStorage.java new file mode 100644 index 0000000..593b049 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotStorage.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.option.SnapshotCopierOptions; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotCopier; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader; +import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter; + +/** + * Snapshot storage. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-08 3:30:05 PM + */ +public interface SnapshotStorage extends Lifecycle, Storage { + + /** + * Set filterBeforeCopyRemote to be true.When true, + * it will filter the data before copy to remote. + */ + boolean setFilterBeforeCopyRemote(); + + /** + * Create a snapshot writer. + */ + SnapshotWriter create(); + + /** + * Open a snapshot reader. + */ + SnapshotReader open(); + + /** + * Copy data from remote uri. + * + * @param uri remote uri + * @param opts copy options + * @return a SnapshotReader instance + */ + SnapshotReader copyFrom(final String uri, final SnapshotCopierOptions opts); + + /** + * Starts a copy job to copy data from remote uri. + * + * @param uri remote uri + * @param opts copy options + * @return a SnapshotCopier instance + */ + SnapshotCopier startToCopyFrom(final String uri, final SnapshotCopierOptions opts); + + /** + * Configure a SnapshotThrottle. + * + * @param snapshotThrottle throttle of snapshot + */ + void setSnapshotThrottle(final SnapshotThrottle snapshotThrottle); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotThrottle.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotThrottle.java new file mode 100644 index 0000000..f454a23 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/SnapshotThrottle.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +/** + * Snapshot throttling during heavy disk reading/writing + * + * @author dennis + */ +public interface SnapshotThrottle { + + /** + * Get available throughput in bytes after throttled + * Must be thread-safe + * + * @param bytes expect size + * @return available size + */ + long throttledByThroughput(final long bytes); +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/Storage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/Storage.java new file mode 100644 index 0000000..41cda79 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/Storage.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage; + +/** + * Common interface for storage. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 3:52:36 PM + */ +public interface Storage { +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorage.java new file mode 100644 index 0000000..04c24cb --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorage.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.impl; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.core.NodeImpl; +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.alipay.sofa.jraft.entity.EnumOutter.ErrorType; +import com.alipay.sofa.jraft.entity.LocalStorageOutter.StablePBMeta; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.RaftMetaStorageOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.RaftMetaStorage; +import com.alipay.sofa.jraft.storage.io.ProtoBufFile; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Raft meta storage,it's not thread-safe. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-26 7:30:36 PM + */ +public class LocalRaftMetaStorage implements RaftMetaStorage { + + private static final Logger LOG = LoggerFactory.getLogger(LocalRaftMetaStorage.class); + private static final String RAFT_META = "raft_meta"; + + private boolean isInited; + private final String path; + private long term; + /** blank votedFor information*/ + private PeerId votedFor = PeerId.emptyPeer(); + private final RaftOptions raftOptions; + private NodeMetrics nodeMetrics; + private NodeImpl node; + + public LocalRaftMetaStorage(final String path, final RaftOptions raftOptions) { + super(); + this.path = path; + this.raftOptions = raftOptions; + } + + @Override + public boolean init(final RaftMetaStorageOptions opts) { + if (this.isInited) { + LOG.warn("Raft meta storage is already inited."); + return true; + } + this.node = opts.getNode(); + this.nodeMetrics = this.node.getNodeMetrics(); + try { + FileUtils.forceMkdir(new File(this.path)); + } catch (final IOException e) { + LOG.error("Fail to mkdir {}", this.path, e); + return false; + } + if (load()) { + this.isInited = true; + return true; + } else { + return false; + } + } + + private boolean load() { + final ProtoBufFile pbFile = newPbFile(); + try { + final StablePBMeta meta = pbFile.load(); + if (meta != null) { + this.term = meta.getTerm(); + return this.votedFor.parse(meta.getVotedfor()); + } + return true; + } catch (final FileNotFoundException e) { + return true; + } catch (final IOException e) { + LOG.error("Fail to load raft meta storage", e); + return false; + } + } + + private ProtoBufFile newPbFile() { + return new ProtoBufFile(this.path + File.separator + RAFT_META); + } + + private boolean save() { + final long start = Utils.monotonicMs(); + final StablePBMeta meta = StablePBMeta.newBuilder() // + .setTerm(this.term) // + .setVotedfor(this.votedFor.toString()) // + .build(); + final ProtoBufFile pbFile = newPbFile(); + try { + if (!pbFile.save(meta, this.raftOptions.isSyncMeta())) { + reportIOError(); + return false; + } + return true; + } catch (final Exception e) { + LOG.error("Fail to save raft meta", e); + reportIOError(); + return false; + } finally { + final long cost = Utils.monotonicMs() - start; + if (this.nodeMetrics != null) { + this.nodeMetrics.recordLatency("save-raft-meta", cost); + } + LOG.info("Save raft meta, path={}, term={}, votedFor={}, cost time={} ms", this.path, this.term, + this.votedFor, cost); + } + } + + private void reportIOError() { + this.node.onError(new RaftException(ErrorType.ERROR_TYPE_META, RaftError.EIO, + "Fail to save raft meta, path=%s", this.path)); + } + + @Override + public void shutdown() { + if (!this.isInited) { + return; + } + save(); + this.isInited = false; + } + + private void checkState() { + if (!this.isInited) { + throw new IllegalStateException("LocalRaftMetaStorage not initialized"); + } + } + + @Override + public boolean setTerm(final long term) { + checkState(); + this.term = term; + return save(); + } + + @Override + public long getTerm() { + checkState(); + return this.term; + } + + @Override + public boolean setVotedFor(final PeerId peerId) { + checkState(); + this.votedFor = peerId; + return save(); + } + + @Override + public PeerId getVotedFor() { + checkState(); + return this.votedFor; + } + + @Override + public boolean setTermAndVotedFor(final long term, final PeerId peerId) { + checkState(); + this.votedFor = peerId; + this.term = term; + return save(); + } + + @Override + public String toString() { + return "RaftMetaStorageImpl [path=" + this.path + ", term=" + this.term + ", votedFor=" + this.votedFor + "]"; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LogManagerImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LogManagerImpl.java new file mode 100644 index 0000000..d8f5100 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/LogManagerImpl.java @@ -0,0 +1,1202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.impl; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.FSMCaller; +import com.alipay.sofa.jraft.Status; +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.conf.ConfigurationManager; +import com.alipay.sofa.jraft.core.NodeMetrics; +import com.alipay.sofa.jraft.entity.EnumOutter.EntryType; +import com.alipay.sofa.jraft.entity.EnumOutter.ErrorType; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.PeerId; +import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta; +import com.alipay.sofa.jraft.error.LogEntryCorruptedException; +import com.alipay.sofa.jraft.error.RaftError; +import com.alipay.sofa.jraft.error.RaftException; +import com.alipay.sofa.jraft.option.LogManagerOptions; +import com.alipay.sofa.jraft.option.LogStorageOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.LogManager; +import com.alipay.sofa.jraft.storage.LogStorage; +import com.alipay.sofa.jraft.util.ArrayDeque; +import com.alipay.sofa.jraft.util.DisruptorBuilder; +import com.alipay.sofa.jraft.util.DisruptorMetricSet; +import com.alipay.sofa.jraft.util.LogExceptionHandler; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.SegmentList; +import com.alipay.sofa.jraft.util.Utils; +import com.lmax.disruptor.EventFactory; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.TimeoutBlockingWaitStrategy; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +/** + * LogManager implementation. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 4:42:20 PM + */ +public class LogManagerImpl implements LogManager { + + private static final Logger LOG = LoggerFactory + .getLogger(LogManagerImpl.class); + + private LogStorage logStorage; + private ConfigurationManager configManager; + private FSMCaller fsmCaller; + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final Lock writeLock = this.lock.writeLock(); + private final Lock readLock = this.lock.readLock(); + private volatile boolean stopped; + private volatile boolean hasError; + private long nextWaitId; + private LogId diskId = new LogId(0, 0); + private LogId appliedId = new LogId(0, 0); + private final SegmentList logsInMemory = new SegmentList<>(true); + private volatile long firstLogIndex; + private volatile long lastLogIndex; + private volatile LogId lastSnapshotId = new LogId(0, 0); + private final Map waitMap = new HashMap<>(); + private Disruptor disruptor; + private RingBuffer diskQueue; + private RaftOptions raftOptions; + private volatile CountDownLatch shutDownLatch; + private NodeMetrics nodeMetrics; + private final CopyOnWriteArrayList lastLogIndexListeners = new CopyOnWriteArrayList<>(); + + private enum EventType { + OTHER, // other event type. + RESET, // reset + TRUNCATE_PREFIX, // truncate log from prefix + TRUNCATE_SUFFIX, // truncate log from suffix + SHUTDOWN, // + LAST_LOG_ID // get last log id + } + + private static class StableClosureEvent { + StableClosure done; + EventType type; + + void reset() { + this.done = null; + this.type = null; + } + } + + private static class StableClosureEventFactory implements EventFactory { + + @Override + public StableClosureEvent newInstance() { + return new StableClosureEvent(); + } + } + + /** + * Waiter metadata + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-04 5:05:04 PM + */ + private static class WaitMeta { + /** callback when new log come in*/ + NewLogCallback onNewLog; + /** callback error code*/ + int errorCode; + /** the waiter pass-in argument */ + Object arg; + + public WaitMeta(final NewLogCallback onNewLog, final Object arg, final int errorCode) { + super(); + this.onNewLog = onNewLog; + this.arg = arg; + this.errorCode = errorCode; + } + + } + + @Override + public void addLastLogIndexListener(final LastLogIndexListener listener) { + this.lastLogIndexListeners.add(listener); + + } + + @Override + public void removeLastLogIndexListener(final LastLogIndexListener listener) { + this.lastLogIndexListeners.remove(listener); + } + + @Override + public boolean init(final LogManagerOptions opts) { + this.writeLock.lock(); + try { + if (opts.getLogStorage() == null) { + LOG.error("Fail to init log manager, log storage is null"); + return false; + } + this.raftOptions = opts.getRaftOptions(); + this.nodeMetrics = opts.getNodeMetrics(); + this.logStorage = opts.getLogStorage(); + this.configManager = opts.getConfigurationManager(); + + LogStorageOptions lsOpts = new LogStorageOptions(); + lsOpts.setConfigurationManager(this.configManager); + lsOpts.setLogEntryCodecFactory(opts.getLogEntryCodecFactory()); + + if (!this.logStorage.init(lsOpts)) { + LOG.error("Fail to init logStorage"); + return false; + } + this.firstLogIndex = this.logStorage.getFirstLogIndex(); + this.lastLogIndex = this.logStorage.getLastLogIndex(); + this.diskId = new LogId(this.lastLogIndex, getTermFromLogStorage(this.lastLogIndex)); + this.fsmCaller = opts.getFsmCaller(); + this.disruptor = DisruptorBuilder. newInstance() // + .setEventFactory(new StableClosureEventFactory()) // + .setRingBufferSize(opts.getDisruptorBufferSize()) // + .setThreadFactory(new NamedThreadFactory("JRaft-LogManager-Disruptor-", true)) // + .setProducerType(ProducerType.MULTI) // + /* + * Use timeout strategy in log manager. If timeout happens, it will called reportError to halt the node. + */ + .setWaitStrategy(new TimeoutBlockingWaitStrategy( + this.raftOptions.getDisruptorPublishEventWaitTimeoutSecs(), TimeUnit.SECONDS)) // + .build(); + this.disruptor.handleEventsWith(new StableClosureEventHandler()); + this.disruptor.setDefaultExceptionHandler(new LogExceptionHandler(this.getClass().getSimpleName(), + (event, ex) -> reportError(-1, "LogManager handle event error"))); + this.diskQueue = this.disruptor.start(); + if (this.nodeMetrics.getMetricRegistry() != null) { + this.nodeMetrics.getMetricRegistry().register("jraft-log-manager-disruptor", + new DisruptorMetricSet(this.diskQueue)); + } + } finally { + this.writeLock.unlock(); + } + return true; + } + + @Override + public boolean hasAvailableCapacityToAppendEntries(final int requiredCapacity) { + if (this.stopped) { + return false; + } + return this.diskQueue.hasAvailableCapacity(requiredCapacity); + } + + private void stopDiskThread() { + this.shutDownLatch = new CountDownLatch(1); + Utils.runInThread(() -> this.diskQueue.publishEvent((event, sequence) -> { + event.reset(); + event.type = EventType.SHUTDOWN; + })); + } + + @Override + public void join() throws InterruptedException { + if (this.shutDownLatch == null) { + return; + } + this.shutDownLatch.await(); + this.disruptor.shutdown(); + } + + @Override + public void shutdown() { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (this.stopped) { + return; + } + this.stopped = true; + doUnlock = false; + wakeupAllWaiter(this.writeLock); + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + stopDiskThread(); + } + + private void clearMemoryLogs(final LogId id) { + this.writeLock.lock(); + try { + this.logsInMemory.removeFromFirstWhen(entry -> entry.getId().compareTo(id) <= 0); + } finally { + this.writeLock.unlock(); + } + } + + private static class LastLogIdClosure extends StableClosure { + + public LastLogIdClosure() { + super(null); + } + + private LogId lastLogId; + + void setLastLogId(final LogId logId) { + Requires.requireTrue(logId.getIndex() == 0 || logId.getTerm() != 0); + this.lastLogId = logId; + } + + private final CountDownLatch latch = new CountDownLatch(1); + + @Override + public void run(final Status status) { + this.latch.countDown(); + } + + void await() throws InterruptedException { + this.latch.await(); + } + + } + + @Override + public void appendEntries(final List entries, final StableClosure done) { + assert(done != null); + + Requires.requireNonNull(done, "done"); + if (this.hasError) { + entries.clear(); + Utils.runClosureInThread(done, new Status(RaftError.EIO, "Corrupted LogStorage")); + return; + } + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (!entries.isEmpty() && !checkAndResolveConflict(entries, done, this.writeLock)) { + // If checkAndResolveConflict returns false, the done will be called in it. + entries.clear(); + return; + } + for (int i = 0; i < entries.size(); i++) { + final LogEntry entry = entries.get(i); + // Set checksum after checkAndResolveConflict + if (this.raftOptions.isEnableLogEntryChecksum()) { + entry.setChecksum(entry.checksum()); + } + if (entry.getType() == EntryType.ENTRY_TYPE_CONFIGURATION) { + Configuration oldConf = new Configuration(); + if (entry.getOldPeers() != null) { + oldConf = new Configuration(entry.getOldPeers(), entry.getOldLearners()); + } + final ConfigurationEntry conf = new ConfigurationEntry(entry.getId(), + new Configuration(entry.getPeers(), entry.getLearners()), oldConf); + this.configManager.add(conf); + } + } + if (!entries.isEmpty()) { + done.setFirstLogIndex(entries.get(0).getId().getIndex()); + this.logsInMemory.addAll(entries); + } + done.setEntries(entries); + + doUnlock = false; + if (!wakeupAllWaiter(this.writeLock)) { + notifyLastLogIndexListeners(); + } + + // publish event out of lock + this.diskQueue.publishEvent((event, sequence) -> { + event.reset(); + event.type = EventType.OTHER; + event.done = done; + }); + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + /** + * Adds event to disk queue, NEVER call it in lock. + * @param done + * @param type + */ + private void offerEvent(final StableClosure done, final EventType type) { + assert(done != null); + + if (this.stopped) { + Utils.runClosureInThread(done, new Status(RaftError.ESTOP, "Log manager is stopped.")); + return; + } + this.diskQueue.publishEvent((event, sequence) -> { + event.reset(); + event.type = type; + event.done = done; + }); + } + + private void notifyLastLogIndexListeners() { + for (int i = 0; i < this.lastLogIndexListeners.size(); i++) { + final LastLogIndexListener listener = this.lastLogIndexListeners.get(i); + if (listener != null) { + try { + listener.onLastLogIndexChanged(this.lastLogIndex); + } catch (final Exception e) { + LOG.error("Fail to notify LastLogIndexListener, listener={}, index={}", listener, this.lastLogIndex); + } + } + } + } + + private boolean wakeupAllWaiter(final Lock lock) { + if (this.waitMap.isEmpty()) { + lock.unlock(); + return false; + } + final List wms = new ArrayList<>(this.waitMap.values()); + final int errCode = this.stopped ? RaftError.ESTOP.getNumber() : RaftError.SUCCESS.getNumber(); + this.waitMap.clear(); + lock.unlock(); + + final int waiterCount = wms.size(); + for (int i = 0; i < waiterCount; i++) { + final WaitMeta wm = wms.get(i); + wm.errorCode = errCode; + Utils.runInThread(() -> runOnNewLog(wm)); + } + return true; + } + + private LogId appendToStorage(final List toAppend) { + LogId lastId = null; + if (!this.hasError) { + final long startMs = Utils.monotonicMs(); + final int entriesCount = toAppend.size(); + this.nodeMetrics.recordSize("append-logs-count", entriesCount); + try { + int writtenSize = 0; + for (int i = 0; i < entriesCount; i++) { + final LogEntry entry = toAppend.get(i); + writtenSize += entry.getData() != null ? entry.getData().remaining() : 0; + } + this.nodeMetrics.recordSize("append-logs-bytes", writtenSize); + final int nAppent = this.logStorage.appendEntries(toAppend); + if (nAppent != entriesCount) { + LOG.error("**Critical error**, fail to appendEntries, nAppent={}, toAppend={}", nAppent, + toAppend.size()); + reportError(RaftError.EIO.getNumber(), "Fail to append log entries"); + } + if (nAppent > 0) { + lastId = toAppend.get(nAppent - 1).getId(); + } + toAppend.clear(); + } finally { + this.nodeMetrics.recordLatency("append-logs", Utils.monotonicMs() - startMs); + } + } + return lastId; + } + + private class AppendBatcher { + List storage; + int cap; + int size; + int bufferSize; + List toAppend; + LogId lastId; + + public AppendBatcher(final List storage, final int cap, final List toAppend, + final LogId lastId) { + super(); + this.storage = storage; + this.cap = cap; + this.toAppend = toAppend; + this.lastId = lastId; + } + + LogId flush() { + if (this.size > 0) { + this.lastId = appendToStorage(this.toAppend); + for (int i = 0; i < this.size; i++) { + this.storage.get(i).getEntries().clear(); + Status st = null; + try { + if (LogManagerImpl.this.hasError) { + st = new Status(RaftError.EIO, "Corrupted LogStorage"); + } else { + st = Status.OK(); + } + this.storage.get(i).run(st); + } catch (Throwable t) { + LOG.error("Fail to run closure with status: {}.", st, t); + } + } + this.toAppend.clear(); + this.storage.clear(); + + } + this.size = 0; + this.bufferSize = 0; + return this.lastId; + } + + void append(final StableClosure done) { + if (this.size == this.cap || this.bufferSize >= LogManagerImpl.this.raftOptions.getMaxAppendBufferSize()) { + flush(); + } + this.storage.add(done); + this.size++; + this.toAppend.addAll(done.getEntries()); + for (final LogEntry entry : done.getEntries()) { + this.bufferSize += entry.getData() != null ? entry.getData().remaining() : 0; + } + } + } + + private class StableClosureEventHandler implements EventHandler { + LogId lastId = LogManagerImpl.this.diskId; + List storage = new ArrayList<>(256); + AppendBatcher ab = new AppendBatcher(this.storage, 256, new ArrayList<>(), + LogManagerImpl.this.diskId); + + @Override + public void onEvent(final StableClosureEvent event, final long sequence, final boolean endOfBatch) + throws Exception { + if (event.type == EventType.SHUTDOWN) { + this.lastId = this.ab.flush(); + setDiskId(this.lastId); + LogManagerImpl.this.shutDownLatch.countDown(); + event.reset(); + return; + } + final StableClosure done = event.done; + final EventType eventType = event.type; + + event.reset(); + + if (done.getEntries() != null && !done.getEntries().isEmpty()) { + this.ab.append(done); + } else { + this.lastId = this.ab.flush(); + boolean ret = true; + switch (eventType) { + case LAST_LOG_ID: + ((LastLogIdClosure) done).setLastLogId(this.lastId.copy()); + break; + case TRUNCATE_PREFIX: + long startMs = Utils.monotonicMs(); + try { + final TruncatePrefixClosure tpc = (TruncatePrefixClosure) done; + LOG.debug("Truncating storage to firstIndexKept={}.", tpc.firstIndexKept); + ret = LogManagerImpl.this.logStorage.truncatePrefix(tpc.firstIndexKept); + } finally { + LogManagerImpl.this.nodeMetrics.recordLatency("truncate-log-prefix", Utils.monotonicMs() + - startMs); + } + break; + case TRUNCATE_SUFFIX: + startMs = Utils.monotonicMs(); + try { + final TruncateSuffixClosure tsc = (TruncateSuffixClosure) done; + LOG.warn("Truncating storage to lastIndexKept={}.", tsc.lastIndexKept); + ret = LogManagerImpl.this.logStorage.truncateSuffix(tsc.lastIndexKept); + if (ret) { + this.lastId.setIndex(tsc.lastIndexKept); + this.lastId.setTerm(tsc.lastTermKept); + Requires.requireTrue(this.lastId.getIndex() == 0 || this.lastId.getTerm() != 0); + } + } finally { + LogManagerImpl.this.nodeMetrics.recordLatency("truncate-log-suffix", Utils.monotonicMs() + - startMs); + } + break; + case RESET: + final ResetClosure rc = (ResetClosure) done; + LOG.info("Resetting storage to nextLogIndex={}.", rc.nextLogIndex); + ret = LogManagerImpl.this.logStorage.reset(rc.nextLogIndex); + break; + default: + break; + } + + if (!ret) { + reportError(RaftError.EIO.getNumber(), "Failed operation in LogStorage"); + } else { + done.run(Status.OK()); + } + } + if (endOfBatch) { + this.lastId = this.ab.flush(); + setDiskId(this.lastId); + } + } + + } + + private void reportError(final int code, final String fmt, final Object... args) { + this.hasError = true; + final RaftException error = new RaftException(ErrorType.ERROR_TYPE_LOG); + error.setStatus(new Status(code, fmt, args)); + this.fsmCaller.onError(error); + } + + private void setDiskId(final LogId id) { + if (id == null) { + return; + } + LogId clearId; + this.writeLock.lock(); + try { + if (id.compareTo(this.diskId) < 0) { + return; + } + this.diskId = id; + clearId = this.diskId.compareTo(this.appliedId) <= 0 ? this.diskId : this.appliedId; + } finally { + this.writeLock.unlock(); + } + if (clearId != null) { + clearMemoryLogs(clearId); + } + } + + @Override + public void setSnapshot(final SnapshotMeta meta) { + LOG.debug("set snapshot: {}.", meta); + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (meta.getLastIncludedIndex() <= this.lastSnapshotId.getIndex()) { + return; + } + final Configuration conf = confFromMeta(meta); + final Configuration oldConf = oldConfFromMeta(meta); + + final ConfigurationEntry entry = new ConfigurationEntry(new LogId(meta.getLastIncludedIndex(), + meta.getLastIncludedTerm()), conf, oldConf); + this.configManager.setSnapshot(entry); + final long term = unsafeGetTerm(meta.getLastIncludedIndex()); + final long savedLastSnapshotIndex = this.lastSnapshotId.getIndex(); + + this.lastSnapshotId.setIndex(meta.getLastIncludedIndex()); + this.lastSnapshotId.setTerm(meta.getLastIncludedTerm()); + + if (this.lastSnapshotId.compareTo(this.appliedId) > 0) { + this.appliedId = this.lastSnapshotId.copy(); + } + // NOTICE: not to update disk_id here as we are not sure if this node really + // has these logs on disk storage. Just leave disk_id as it was, which can keep + // these logs in memory all the time until they are flushed to disk. By this + // way we can avoid some corner cases which failed to get logs. + // See https://github.com/baidu/braft/pull/224/commits/8ef6fdbf70d23f5a4ee147356a889e2c0fa22aac + // if (this.lastSnapshotId.compareTo(this.diskId) > 0) { + // this.diskId = this.lastSnapshotId.copy(); + // } + + if (term == 0) { + // last_included_index is larger than last_index + // FIXME: what if last_included_index is less than first_index? + doUnlock = false; + //unlock in truncatePrefix + truncatePrefix(meta.getLastIncludedIndex() + 1, this.writeLock); + } else if (term == meta.getLastIncludedTerm()) { + // Truncating log to the index of the last snapshot. + // We don't truncate log before the last snapshot immediately since + // some log around last_snapshot_index is probably needed by some + // followers + // TODO if there are still be need? + if (savedLastSnapshotIndex > 0) { + doUnlock = false; + //unlock in truncatePrefix + truncatePrefix(savedLastSnapshotIndex + 1, this.writeLock); + } + } else { + if (!reset(meta.getLastIncludedIndex() + 1)) { + LOG.warn("Reset log manager failed, nextLogIndex={}.", meta.getLastIncludedIndex() + 1); + } + } + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + + } + + private Configuration oldConfFromMeta(final SnapshotMeta meta) { + final Configuration oldConf = new Configuration(); + for (int i = 0; i < meta.getOldPeersCount(); i++) { + final PeerId peer = new PeerId(); + peer.parse(meta.getOldPeers(i)); + oldConf.addPeer(peer); + } + for (int i = 0; i < meta.getOldLearnersCount(); i++) { + final PeerId peer = new PeerId(); + peer.parse(meta.getOldLearners(i)); + oldConf.addLearner(peer); + } + return oldConf; + } + + private Configuration confFromMeta(final SnapshotMeta meta) { + final Configuration conf = new Configuration(); + for (int i = 0; i < meta.getPeersCount(); i++) { + final PeerId peer = new PeerId(); + peer.parse(meta.getPeers(i)); + conf.addPeer(peer); + } + for (int i = 0; i < meta.getLearnersCount(); i++) { + final PeerId peer = new PeerId(); + peer.parse(meta.getLearners(i)); + conf.addLearner(peer); + } + return conf; + } + + @Override + public void clearBufferedLogs() { + boolean doUnlock = true; + this.writeLock.lock(); + try { + if (this.lastSnapshotId.getIndex() != 0) { + doUnlock = false; + //unlock in truncatePrefix + truncatePrefix(this.lastSnapshotId.getIndex() + 1, this.writeLock); + } + } finally { + if (doUnlock) { + this.writeLock.unlock(); + } + } + } + + private String descLogsInMemory() { + final StringBuilder sb = new StringBuilder(); + boolean wasFirst = true; + for (int i = 0; i < this.logsInMemory.size(); i++) { + LogEntry logEntry = this.logsInMemory.get(i); + if (!wasFirst) { + sb.append(","); + } else { + wasFirst = false; + } + sb.append(""); + } + return sb.toString(); + } + + protected LogEntry getEntryFromMemory(final long index) { + LogEntry entry = null; + if (!this.logsInMemory.isEmpty()) { + final long firstIndex = this.logsInMemory.peekFirst().getId().getIndex(); + final long lastIndex = this.logsInMemory.peekLast().getId().getIndex(); + if (lastIndex - firstIndex + 1 != this.logsInMemory.size()) { + throw new IllegalStateException(String.format("lastIndex=%d,firstIndex=%d,logsInMemory=[%s]", + lastIndex, firstIndex, descLogsInMemory())); + } + if (index >= firstIndex && index <= lastIndex) { + entry = this.logsInMemory.get((int) (index - firstIndex)); + } + } + return entry; + } + + @Override + public LogEntry getEntry(final long index) { + this.readLock.lock(); + try { + if (index > this.lastLogIndex || index < this.firstLogIndex) { + return null; + } + final LogEntry entry = getEntryFromMemory(index); + if (entry != null) { + return entry; + } + } finally { + this.readLock.unlock(); + } + final LogEntry entry = this.logStorage.getEntry(index); + if (entry == null) { + reportError(RaftError.EIO.getNumber(), "Corrupted entry at index=%d, not found", index); + } + // Validate checksum + if (entry != null && this.raftOptions.isEnableLogEntryChecksum() && entry.isCorrupted()) { + String msg = String.format("Corrupted entry at index=%d, term=%d, expectedChecksum=%d, realChecksum=%d", + index, entry.getId().getTerm(), entry.getChecksum(), entry.checksum()); + // Report error to node and throw exception. + reportError(RaftError.EIO.getNumber(), msg); + throw new LogEntryCorruptedException(msg); + } + return entry; + } + + @Override + public long getTerm(final long index) { + if (index == 0) { + return 0; + } + this.readLock.lock(); + try { + // check index equal snapshot_index, return snapshot_term + if (index == this.lastSnapshotId.getIndex()) { + return this.lastSnapshotId.getTerm(); + } + // out of range, direct return 0 + if (index > this.lastLogIndex || index < this.firstLogIndex) { + return 0; + } + final LogEntry entry = getEntryFromMemory(index); + if (entry != null) { + return entry.getId().getTerm(); + } + } finally { + this.readLock.unlock(); + } + return getTermFromLogStorage(index); + } + + private long getTermFromLogStorage(final long index) { + final LogEntry entry = this.logStorage.getEntry(index); + if (entry != null) { + if (this.raftOptions.isEnableLogEntryChecksum() && entry.isCorrupted()) { + // Report error to node and throw exception. + final String msg = String.format( + "The log entry is corrupted, index=%d, term=%d, expectedChecksum=%d, realChecksum=%d", entry + .getId().getIndex(), entry.getId().getTerm(), entry.getChecksum(), entry.checksum()); + reportError(RaftError.EIO.getNumber(), msg); + throw new LogEntryCorruptedException(msg); + } + + return entry.getId().getTerm(); + } + return 0; + } + + @Override + public long getFirstLogIndex() { + this.readLock.lock(); + try { + return this.firstLogIndex; + } finally { + this.readLock.unlock(); + } + } + + @Override + public long getLastLogIndex() { + return getLastLogIndex(false); + } + + @Override + public long getLastLogIndex(final boolean isFlush) { + LastLogIdClosure c; + this.readLock.lock(); + try { + if (!isFlush) { + return this.lastLogIndex; + } else { + if (this.lastLogIndex == this.lastSnapshotId.getIndex()) { + return this.lastLogIndex; + } + c = new LastLogIdClosure(); + } + } finally { + this.readLock.unlock(); + } + offerEvent(c, EventType.LAST_LOG_ID); + try { + c.await(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return c.lastLogId.getIndex(); + } + + private long unsafeGetTerm(final long index) { + if (index == 0) { + return 0; + } + + final LogId lss = this.lastSnapshotId; + if (index == lss.getIndex()) { + return lss.getTerm(); + } + if (index > this.lastLogIndex || index < this.firstLogIndex) { + return 0; + } + final LogEntry entry = getEntryFromMemory(index); + if (entry != null) { + return entry.getId().getTerm(); + } + return getTermFromLogStorage(index); + } + + @Override + public LogId getLastLogId(final boolean isFlush) { + LastLogIdClosure c; + this.readLock.lock(); + try { + if (!isFlush) { + if (this.lastLogIndex >= this.firstLogIndex) { + return new LogId(this.lastLogIndex, unsafeGetTerm(this.lastLogIndex)); + } + return this.lastSnapshotId; + } else { + if (this.lastLogIndex == this.lastSnapshotId.getIndex()) { + return this.lastSnapshotId; + } + c = new LastLogIdClosure(); + } + } finally { + this.readLock.unlock(); + } + offerEvent(c, EventType.LAST_LOG_ID); + try { + c.await(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return c.lastLogId; + } + + private static class TruncatePrefixClosure extends StableClosure { + long firstIndexKept; + + public TruncatePrefixClosure(final long firstIndexKept) { + super(null); + this.firstIndexKept = firstIndexKept; + } + + @Override + public void run(final Status status) { + + } + + } + + private static class TruncateSuffixClosure extends StableClosure { + long lastIndexKept; + long lastTermKept; + + public TruncateSuffixClosure(final long lastIndexKept, final long lastTermKept) { + super(null); + this.lastIndexKept = lastIndexKept; + this.lastTermKept = lastTermKept; + } + + @Override + public void run(final Status status) { + + } + + } + + private static class ResetClosure extends StableClosure { + long nextLogIndex; + + public ResetClosure(final long nextLogIndex) { + super(null); + this.nextLogIndex = nextLogIndex; + } + + @Override + public void run(final Status status) { + + } + } + + private boolean truncatePrefix(final long firstIndexKept, final Lock lock) { + + this.logsInMemory.removeFromFirstWhen(entry -> entry.getId().getIndex() < firstIndexKept); + + // TODO maybe it's fine here + Requires.requireTrue(firstIndexKept >= this.firstLogIndex, + "Try to truncate logs before %d, but the firstLogIndex is %d", firstIndexKept, this.firstLogIndex); + + this.firstLogIndex = firstIndexKept; + if (firstIndexKept > this.lastLogIndex) { + // The entry log is dropped + this.lastLogIndex = firstIndexKept - 1; + } + LOG.debug("Truncate prefix, firstIndexKept is :{}", firstIndexKept); + this.configManager.truncatePrefix(firstIndexKept); + lock.unlock(); + final TruncatePrefixClosure c = new TruncatePrefixClosure(firstIndexKept); + offerEvent(c, EventType.TRUNCATE_PREFIX); + return true; + } + + private boolean reset(final long nextLogIndex) { + this.writeLock.lock(); + try { + this.logsInMemory.clear(); + this.firstLogIndex = nextLogIndex; + this.lastLogIndex = nextLogIndex - 1; + this.configManager.truncatePrefix(this.firstLogIndex); + this.configManager.truncateSuffix(this.lastLogIndex); + return true; + } finally { + this.writeLock.unlock(); + final ResetClosure c = new ResetClosure(nextLogIndex); + offerEvent(c, EventType.RESET); + } + + } + + private void unsafeTruncateSuffix(final long lastIndexKept, final Lock lock) { + if (lastIndexKept < this.appliedId.getIndex()) { + LOG.error("FATAL ERROR: Can't truncate logs before appliedId={}, lastIndexKept={}", this.appliedId, + lastIndexKept); + return; + } + + this.logsInMemory.removeFromLastWhen(entry -> entry.getId().getIndex() > lastIndexKept); + + this.lastLogIndex = lastIndexKept; + final long lastTermKept = unsafeGetTerm(lastIndexKept); + Requires.requireTrue(this.lastLogIndex == 0 || lastTermKept != 0); + LOG.debug("Truncate suffix :{}", lastIndexKept); + this.configManager.truncateSuffix(lastIndexKept); + lock.unlock(); + final TruncateSuffixClosure c = new TruncateSuffixClosure(lastIndexKept, lastTermKept); + offerEvent(c, EventType.TRUNCATE_SUFFIX); + lock.lock(); + } + + @SuppressWarnings("NonAtomicOperationOnVolatileField") + private boolean checkAndResolveConflict(final List entries, final StableClosure done, final Lock lock) { + final LogEntry firstLogEntry = ArrayDeque.peekFirst(entries); + if (firstLogEntry.getId().getIndex() == 0) { + // Node is currently the leader and |entries| are from the user who + // don't know the correct indexes the logs should assign to. So we have + // to assign indexes to the appending entries + for (int i = 0; i < entries.size(); i++) { + entries.get(i).getId().setIndex(++this.lastLogIndex); + } + return true; + } else { + // Node is currently a follower and |entries| are from the leader. We + // should check and resolve the conflicts between the local logs and + // |entries| + if (firstLogEntry.getId().getIndex() > this.lastLogIndex + 1) { + Utils.runClosureInThread(done, new Status(RaftError.EINVAL, + "There's gap between first_index=%d and last_log_index=%d", firstLogEntry.getId().getIndex(), + this.lastLogIndex)); + return false; + } + final long appliedIndex = this.appliedId.getIndex(); + final LogEntry lastLogEntry = ArrayDeque.peekLast(entries); + if (lastLogEntry.getId().getIndex() <= appliedIndex) { + LOG.warn( + "Received entries of which the lastLog={} is not greater than appliedIndex={}, return immediately with nothing changed.", + lastLogEntry.getId().getIndex(), appliedIndex); + // Replicate old logs before appliedIndex should be considered successfully, response OK. + Utils.runClosureInThread(done); + return false; + } + if (firstLogEntry.getId().getIndex() == this.lastLogIndex + 1) { + // fast path + this.lastLogIndex = lastLogEntry.getId().getIndex(); + } else { + // Appending entries overlap the local ones. We should find if there + // is a conflicting index from which we should truncate the local + // ones. + int conflictingIndex = 0; + for (; conflictingIndex < entries.size(); conflictingIndex++) { + if (unsafeGetTerm(entries.get(conflictingIndex).getId().getIndex()) != entries + .get(conflictingIndex).getId().getTerm()) { + break; + } + } + if (conflictingIndex != entries.size()) { + if (entries.get(conflictingIndex).getId().getIndex() <= this.lastLogIndex) { + // Truncate all the conflicting entries to make local logs + // consensus with the leader. + unsafeTruncateSuffix(entries.get(conflictingIndex).getId().getIndex() - 1, lock); + } + this.lastLogIndex = lastLogEntry.getId().getIndex(); + } // else this is a duplicated AppendEntriesRequest, we have + // nothing to do besides releasing all the entries + if (conflictingIndex > 0) { + // Remove duplication + entries.subList(0, conflictingIndex).clear(); + } + } + return true; + } + } + + @Override + public ConfigurationEntry getConfiguration(final long index) { + this.readLock.lock(); + try { + return this.configManager.get(index); + } finally { + this.readLock.unlock(); + } + } + + @Override + public ConfigurationEntry checkAndSetConfiguration(final ConfigurationEntry current) { + if (current == null) { + return null; + } + this.readLock.lock(); + try { + final ConfigurationEntry lastConf = this.configManager.getLastConfiguration(); + if (lastConf != null && !lastConf.isEmpty() && !current.getId().equals(lastConf.getId())) { + return lastConf; + } + } finally { + this.readLock.unlock(); + } + return current; + } + + @Override + public long wait(final long expectedLastLogIndex, final NewLogCallback cb, final Object arg) { + final WaitMeta wm = new WaitMeta(cb, arg, 0); + return notifyOnNewLog(expectedLastLogIndex, wm); + } + + private long notifyOnNewLog(final long expectedLastLogIndex, final WaitMeta wm) { + this.writeLock.lock(); + try { + if (expectedLastLogIndex != this.lastLogIndex || this.stopped) { + wm.errorCode = this.stopped ? RaftError.ESTOP.getNumber() : 0; + Utils.runInThread(() -> runOnNewLog(wm)); + return 0L; + } + if (this.nextWaitId == 0) { //skip 0 + ++this.nextWaitId; + } + final long waitId = this.nextWaitId++; + this.waitMap.put(waitId, wm); + return waitId; + } finally { + this.writeLock.unlock(); + } + } + + @Override + public boolean removeWaiter(final long id) { + this.writeLock.lock(); + try { + return this.waitMap.remove(id) != null; + } finally { + this.writeLock.unlock(); + } + } + + @Override + public void setAppliedId(final LogId appliedId) { + LogId clearId; + this.writeLock.lock(); + try { + if (appliedId.compareTo(this.appliedId) < 0) { + return; + } + this.appliedId = appliedId.copy(); + clearId = this.diskId.compareTo(this.appliedId) <= 0 ? this.diskId : this.appliedId; + } finally { + this.writeLock.unlock(); + } + if (clearId != null) { + clearMemoryLogs(clearId); + } + } + + void runOnNewLog(final WaitMeta wm) { + wm.onNewLog.onNewLog(wm.arg, wm.errorCode); + } + + @Override + public Status checkConsistency() { + this.readLock.lock(); + try { + Requires.requireTrue(this.firstLogIndex > 0); + Requires.requireTrue(this.lastLogIndex >= 0); + if (this.lastSnapshotId.equals(new LogId(0, 0))) { + if (this.firstLogIndex == 1) { + return Status.OK(); + } + return new Status(RaftError.EIO, "Missing logs in (0, %d)", this.firstLogIndex); + } else { + if (this.lastSnapshotId.getIndex() >= this.firstLogIndex - 1 + && this.lastSnapshotId.getIndex() <= this.lastLogIndex) { + return Status.OK(); + } + return new Status(RaftError.EIO, "There's a gap between snapshot={%d, %d} and log=[%d, %d] ", + this.lastSnapshotId.toString(), this.lastSnapshotId.getTerm(), this.firstLogIndex, + this.lastLogIndex); + } + } finally { + this.readLock.unlock(); + } + } + + @Override + public void describe(final Printer out) { + final long _firstLogIndex; + final long _lastLogIndex; + final String _diskId; + final String _appliedId; + final String _lastSnapshotId; + this.readLock.lock(); + try { + _firstLogIndex = this.firstLogIndex; + _lastLogIndex = this.lastLogIndex; + _diskId = String.valueOf(this.diskId); + _appliedId = String.valueOf(this.appliedId); + _lastSnapshotId = String.valueOf(this.lastSnapshotId); + } finally { + this.readLock.unlock(); + } + out.print(" storage: [") // + .print(_firstLogIndex) // + .print(", ") // + .print(_lastLogIndex) // + .println(']'); + out.print(" diskId: ") // + .println(_diskId); + out.print(" appliedId: ") // + .println(_appliedId); + out.print(" lastSnapshotId: ") // + .println(_lastSnapshotId); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorage.java new file mode 100644 index 0000000..4d22c4d --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorage.java @@ -0,0 +1,764 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.impl; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.conf.Configuration; +import com.alipay.sofa.jraft.conf.ConfigurationEntry; +import com.alipay.sofa.jraft.conf.ConfigurationManager; +import com.alipay.sofa.jraft.entity.EnumOutter.EntryType; +import com.alipay.sofa.jraft.entity.LogEntry; +import com.alipay.sofa.jraft.entity.LogId; +import com.alipay.sofa.jraft.entity.codec.LogEntryDecoder; +import com.alipay.sofa.jraft.entity.codec.LogEntryEncoder; +import com.alipay.sofa.jraft.option.LogStorageOptions; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.LogStorage; +import com.alipay.sofa.jraft.util.Bits; +import com.alipay.sofa.jraft.util.BytesUtil; +import com.alipay.sofa.jraft.util.DebugStatistics; +import com.alipay.sofa.jraft.util.Describer; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.StorageOptionsFactory; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Log storage based on rocksdb. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-06 7:27:47 AM + */ +public class RocksDBLogStorage implements LogStorage, Describer { + + private static final Logger LOG = LoggerFactory.getLogger(RocksDBLogStorage.class); + + static { + RocksDB.loadLibrary(); + } + + /** + * Write batch template. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2017-Nov-08 11:19:22 AM + */ + private interface WriteBatchTemplate { + + void execute(WriteBatch batch) throws RocksDBException, IOException, InterruptedException; + } + + /** + * A write context + * @author boyan(boyan@antfin.com) + * + */ + public interface WriteContext { + /** + * Start a sub job. + */ + default void startJob() { + } + + /** + * Finish a sub job + */ + default void finishJob() { + } + + /** + * Adds a callback that will be invoked after all sub jobs finish. + */ + default void addFinishHook(final Runnable r) { + + } + + /** + * Set an exception to context. + * @param e exception + */ + default void setError(final Exception e) { + } + + /** + * Wait for all sub jobs finish. + */ + default void joinAll() throws InterruptedException, IOException { + } + } + + /** + * An empty write context + * @author boyan(boyan@antfin.com) + * + */ + protected static class EmptyWriteContext implements WriteContext { + static EmptyWriteContext INSTANCE = new EmptyWriteContext(); + } + + private final String path; + private final boolean sync; + private final boolean openStatistics; + private RocksDB db; + private DBOptions dbOptions; + private WriteOptions writeOptions; + private final List cfOptions = new ArrayList<>(); + private ColumnFamilyHandle defaultHandle; + private ColumnFamilyHandle confHandle; + private ReadOptions totalOrderReadOptions; + private DebugStatistics statistics; + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final Lock readLock = this.readWriteLock.readLock(); + private final Lock writeLock = this.readWriteLock.writeLock(); + + private volatile long firstLogIndex = 1; + + private volatile boolean hasLoadFirstLogIndex; + + private LogEntryEncoder logEntryEncoder; + private LogEntryDecoder logEntryDecoder; + + public RocksDBLogStorage(final String path, final RaftOptions raftOptions) { + super(); + this.path = path; + this.sync = raftOptions.isSync(); + this.openStatistics = raftOptions.isOpenStatistics(); + } + + public static DBOptions createDBOptions() { + return StorageOptionsFactory.getRocksDBOptions(RocksDBLogStorage.class); + } + + public static ColumnFamilyOptions createColumnFamilyOptions() { + final BlockBasedTableConfig tConfig = StorageOptionsFactory + .getRocksDBTableFormatConfig(RocksDBLogStorage.class); + return StorageOptionsFactory.getRocksDBColumnFamilyOptions(RocksDBLogStorage.class) // + .useFixedLengthPrefixExtractor(8) // + .setTableFormatConfig(tConfig) // + .setMergeOperator(new StringAppendOperator()); + } + + @Override + public boolean init(final LogStorageOptions opts) { + Requires.requireNonNull(opts.getConfigurationManager(), "Null conf manager"); + Requires.requireNonNull(opts.getLogEntryCodecFactory(), "Null log entry codec factory"); + this.writeLock.lock(); + try { + if (this.db != null) { + LOG.warn("RocksDBLogStorage init() in {} already.", this.path); + return true; + } + this.logEntryDecoder = opts.getLogEntryCodecFactory().decoder(); + this.logEntryEncoder = opts.getLogEntryCodecFactory().encoder(); + Requires.requireNonNull(this.logEntryDecoder, "Null log entry decoder"); + Requires.requireNonNull(this.logEntryEncoder, "Null log entry encoder"); + this.dbOptions = createDBOptions(); + if (this.openStatistics) { + this.statistics = new DebugStatistics(); + this.dbOptions.setStatistics(this.statistics); + } + + this.writeOptions = new WriteOptions(); + this.writeOptions.setSync(this.sync); + this.totalOrderReadOptions = new ReadOptions(); + this.totalOrderReadOptions.setTotalOrderSeek(true); + + return initAndLoad(opts.getConfigurationManager()); + } catch (final RocksDBException e) { + LOG.error("Fail to init RocksDBLogStorage, path={}.", this.path, e); + return false; + } finally { + this.writeLock.unlock(); + } + + } + + private boolean initAndLoad(final ConfigurationManager confManager) throws RocksDBException { + this.hasLoadFirstLogIndex = false; + this.firstLogIndex = 1; + final List columnFamilyDescriptors = new ArrayList<>(); + final ColumnFamilyOptions cfOption = createColumnFamilyOptions(); + this.cfOptions.add(cfOption); + // Column family to store configuration log entry. + columnFamilyDescriptors.add(new ColumnFamilyDescriptor("Configuration".getBytes(), cfOption)); + // Default column family to store user data log entry. + columnFamilyDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOption)); + + openDB(columnFamilyDescriptors); + load(confManager); + return onInitLoaded(); + } + + /** + * First log index and last log index key in configuration column family. + */ + public static final byte[] FIRST_LOG_IDX_KEY = Utils.getBytes("meta/firstLogIndex"); + + private void load(final ConfigurationManager confManager) { + checkState(); + try (final RocksIterator it = this.db.newIterator(this.confHandle, this.totalOrderReadOptions)) { + it.seekToFirst(); + while (it.isValid()) { + final byte[] ks = it.key(); + final byte[] bs = it.value(); + + // LogEntry index + if (ks.length == 8) { + final LogEntry entry = this.logEntryDecoder.decode(bs); + if (entry != null) { + if (entry.getType() == EntryType.ENTRY_TYPE_CONFIGURATION) { + final ConfigurationEntry confEntry = new ConfigurationEntry(); + confEntry.setId(new LogId(entry.getId().getIndex(), entry.getId().getTerm())); + confEntry.setConf(new Configuration(entry.getPeers(), entry.getLearners())); + if (entry.getOldPeers() != null) { + confEntry.setOldConf(new Configuration(entry.getOldPeers(), entry.getOldLearners())); + } + if (confManager != null) { + confManager.add(confEntry); + } + } + } else { + LOG.warn("Fail to decode conf entry at index {}, the log data is: {}.", Bits.getLong(ks, 0), + BytesUtil.toHex(bs)); + } + } else { + if (Arrays.equals(FIRST_LOG_IDX_KEY, ks)) { + setFirstLogIndex(Bits.getLong(bs, 0)); + truncatePrefixInBackground(0L, this.firstLogIndex); + } else { + LOG.warn("Unknown entry in configuration storage key={}, value={}.", BytesUtil.toHex(ks), + BytesUtil.toHex(bs)); + } + } + it.next(); + } + } + } + + private void setFirstLogIndex(final long index) { + this.firstLogIndex = index; + this.hasLoadFirstLogIndex = true; + } + + /** + * Save the first log index into conf column family. + */ + private boolean saveFirstLogIndex(final long firstLogIndex) { + this.readLock.lock(); + try { + final byte[] vs = new byte[8]; + Bits.putLong(vs, 0, firstLogIndex); + checkState(); + this.db.put(this.confHandle, this.writeOptions, FIRST_LOG_IDX_KEY, vs); + return true; + } catch (final RocksDBException e) { + LOG.error("Fail to save first log index {} in {}.", firstLogIndex, this.path, e); + return false; + } finally { + this.readLock.unlock(); + } + } + + private void openDB(final List columnFamilyDescriptors) throws RocksDBException { + final List columnFamilyHandles = new ArrayList<>(); + + final File dir = new File(this.path); + if (dir.exists() && !dir.isDirectory()) { + throw new IllegalStateException("Invalid log path, it's a regular file: " + this.path); + } + this.db = RocksDB.open(this.dbOptions, this.path, columnFamilyDescriptors, columnFamilyHandles); + + assert (columnFamilyHandles.size() == 2); + this.confHandle = columnFamilyHandles.get(0); + this.defaultHandle = columnFamilyHandles.get(1); + } + + private void checkState() { + Requires.requireNonNull(this.db, "DB not initialized or destroyed"); + } + + /** + * Execute write batch template. + * + * @param template write batch template + */ + private boolean executeBatch(final WriteBatchTemplate template) { + this.readLock.lock(); + if (this.db == null) { + LOG.warn("DB not initialized or destroyed in data path: {}.", this.path); + this.readLock.unlock(); + return false; + } + try (final WriteBatch batch = new WriteBatch()) { + template.execute(batch); + this.db.write(this.writeOptions, batch); + } catch (final RocksDBException e) { + LOG.error("Execute batch failed with rocksdb exception.", e); + return false; + } catch (final IOException e) { + LOG.error("Execute batch failed with io exception.", e); + return false; + } catch (final InterruptedException e) { + LOG.error("Execute batch failed with interrupt.", e); + Thread.currentThread().interrupt(); + return false; + } finally { + this.readLock.unlock(); + } + return true; + } + + @Override + public void shutdown() { + this.writeLock.lock(); + try { + // The shutdown order is matter. + // 1. close column family handles + closeDB(); + onShutdown(); + // 2. close column family options. + for (final ColumnFamilyOptions opt : this.cfOptions) { + opt.close(); + } + // 3. close options + this.dbOptions.close(); + if (this.statistics != null) { + this.statistics.close(); + } + this.writeOptions.close(); + this.totalOrderReadOptions.close(); + // 4. help gc. + this.cfOptions.clear(); + this.dbOptions = null; + this.statistics = null; + this.writeOptions = null; + this.totalOrderReadOptions = null; + this.defaultHandle = null; + this.confHandle = null; + this.db = null; + LOG.info("DB destroyed, the db path is: {}.", this.path); + } finally { + this.writeLock.unlock(); + } + } + + private void closeDB() { + this.confHandle.close(); + this.defaultHandle.close(); + this.db.close(); + } + + @Override + public long getFirstLogIndex() { + this.readLock.lock(); + RocksIterator it = null; + try { + if (this.hasLoadFirstLogIndex) { + return this.firstLogIndex; + } + checkState(); + it = this.db.newIterator(this.defaultHandle, this.totalOrderReadOptions); + it.seekToFirst(); + if (it.isValid()) { + final long ret = Bits.getLong(it.key(), 0); + saveFirstLogIndex(ret); + setFirstLogIndex(ret); + return ret; + } + return 1L; + } finally { + if (it != null) { + it.close(); + } + this.readLock.unlock(); + } + } + + @Override + public long getLastLogIndex() { + this.readLock.lock(); + checkState(); + try (final RocksIterator it = this.db.newIterator(this.defaultHandle, this.totalOrderReadOptions)) { + it.seekToLast(); + if (it.isValid()) { + return Bits.getLong(it.key(), 0); + } + return 0L; + } finally { + this.readLock.unlock(); + } + } + + @Override + public LogEntry getEntry(final long index) { + this.readLock.lock(); + try { + if (this.hasLoadFirstLogIndex && index < this.firstLogIndex) { + return null; + } + return getEntryFromDB(index); + } catch (final RocksDBException | IOException e) { + LOG.error("Fail to get log entry at index {} in data path: {}.", index, this.path, e); + } finally { + this.readLock.unlock(); + } + return null; + } + + @OnlyForTest + LogEntry getEntryFromDB(final long index) throws IOException, RocksDBException { + final byte[] keyBytes = getKeyBytes(index); + final byte[] bs = onDataGet(index, getValueFromRocksDB(keyBytes)); + if (bs != null) { + final LogEntry entry = this.logEntryDecoder.decode(bs); + if (entry != null) { + return entry; + } else { + LOG.error("Bad log entry format for index={}, the log data is: {}.", index, BytesUtil.toHex(bs)); + // invalid data remove? TODO + return null; + } + } + return null; + } + + protected byte[] getValueFromRocksDB(final byte[] keyBytes) throws RocksDBException { + checkState(); + return this.db.get(this.defaultHandle, keyBytes); + } + + protected byte[] getKeyBytes(final long index) { + final byte[] ks = new byte[8]; + Bits.putLong(ks, 0, index); + return ks; + } + + @Override + public long getTerm(final long index) { + final LogEntry entry = getEntry(index); + if (entry != null) { + return entry.getId().getTerm(); + } + return 0; + } + + private void addConfBatch(final LogEntry entry, final WriteBatch batch) throws RocksDBException { + final byte[] ks = getKeyBytes(entry.getId().getIndex()); + final byte[] content = this.logEntryEncoder.encode(entry); + batch.put(this.defaultHandle, ks, content); + batch.put(this.confHandle, ks, content); + } + + private void addDataBatch(final LogEntry entry, final WriteBatch batch, + final WriteContext ctx) throws RocksDBException, IOException, InterruptedException { + final long logIndex = entry.getId().getIndex(); + final byte[] content = this.logEntryEncoder.encode(entry); + batch.put(this.defaultHandle, getKeyBytes(logIndex), onDataAppend(logIndex, content, ctx)); + } + + @Override + public boolean appendEntry(final LogEntry entry) { + if (entry.getType() == EntryType.ENTRY_TYPE_CONFIGURATION) { + return executeBatch(batch -> addConfBatch(entry, batch)); + } else { + this.readLock.lock(); + try { + if (this.db == null) { + LOG.warn("DB not initialized or destroyed in data path: {}.", this.path); + return false; + } + final WriteContext writeCtx = newWriteContext(); + final long logIndex = entry.getId().getIndex(); + final byte[] valueBytes = this.logEntryEncoder.encode(entry); + final byte[] newValueBytes = onDataAppend(logIndex, valueBytes, writeCtx); + writeCtx.startJob(); + this.db.put(this.defaultHandle, this.writeOptions, getKeyBytes(logIndex), newValueBytes); + writeCtx.joinAll(); + if (newValueBytes != valueBytes) { + doSync(); + } + return true; + } catch (final RocksDBException | IOException e) { + LOG.error("Fail to append entry.", e); + return false; + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } finally { + this.readLock.unlock(); + } + } + } + + private void doSync() throws IOException, InterruptedException { + onSync(); + } + + @Override + public int appendEntries(final List entries) { + if (entries == null || entries.isEmpty()) { + return 0; + } + final int entriesCount = entries.size(); + final boolean ret = executeBatch(batch -> { + final WriteContext writeCtx = newWriteContext(); + for (int i = 0; i < entriesCount; i++) { + final LogEntry entry = entries.get(i); + if (entry.getType() == EntryType.ENTRY_TYPE_CONFIGURATION) { + addConfBatch(entry, batch); + } else { + writeCtx.startJob(); + addDataBatch(entry, batch, writeCtx); + } + } + writeCtx.joinAll(); + doSync(); + }); + + if (ret) { + return entriesCount; + } else { + return 0; + } + } + + @Override + public boolean truncatePrefix(final long firstIndexKept) { + this.readLock.lock(); + try { + final long startIndex = getFirstLogIndex(); + final boolean ret = saveFirstLogIndex(firstIndexKept); + if (ret) { + setFirstLogIndex(firstIndexKept); + } + truncatePrefixInBackground(startIndex, firstIndexKept); + return ret; + } finally { + this.readLock.unlock(); + } + + } + + private void truncatePrefixInBackground(final long startIndex, final long firstIndexKept) { + // delete logs in background. + Utils.runInThread(() -> { + long startMs = Utils.monotonicMs(); + this.readLock.lock(); + try { + RocksDB db = this.db; + if (db == null) { + LOG.warn( + "DB is null while truncating prefixed logs in data path: {}, the range is: [{}, {})", + this.path, startIndex, firstIndexKept); + return; + } + onTruncatePrefix(startIndex, firstIndexKept); + // Note https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys + final byte[] startKey = getKeyBytes(startIndex); + final byte[] endKey = getKeyBytes(firstIndexKept); + // deleteRange to delete all keys in range. + db.deleteRange(this.defaultHandle, startKey, endKey); + db.deleteRange(this.confHandle, startKey, endKey); + // deleteFilesInRanges to speedup reclaiming disk space on write-heavy load. + db.deleteFilesInRanges(this.defaultHandle, Arrays.asList(startKey, endKey), false); + db.deleteFilesInRanges(this.confHandle, Arrays.asList(startKey, endKey), false); + } catch (final RocksDBException | IOException e) { + LOG.error("Fail to truncatePrefix in data path: {}, firstIndexKept={}.", this.path, firstIndexKept, e); + } finally { + this.readLock.unlock(); + LOG.info("Truncated prefix logs in data path: {} from log index {} to {}, cost {} ms.", + this.path, startIndex, firstIndexKept, Utils.monotonicMs() - startMs); + } + }); + } + + @Override + public boolean truncateSuffix(final long lastIndexKept) { + this.readLock.lock(); + try { + try { + onTruncateSuffix(lastIndexKept); + } finally { + this.db.deleteRange(this.defaultHandle, this.writeOptions, getKeyBytes(lastIndexKept + 1), + getKeyBytes(getLastLogIndex() + 1)); + this.db.deleteRange(this.confHandle, this.writeOptions, getKeyBytes(lastIndexKept + 1), + getKeyBytes(getLastLogIndex() + 1)); + } + return true; + } catch (final RocksDBException | IOException e) { + LOG.error("Fail to truncateSuffix {} in data path: {}.", lastIndexKept, this.path, e); + } finally { + this.readLock.unlock(); + } + return false; + } + + @Override + public boolean reset(final long nextLogIndex) { + if (nextLogIndex <= 0) { + throw new IllegalArgumentException("Invalid next log index."); + } + this.writeLock.lock(); + try (final Options opt = new Options()) { + LogEntry entry = getEntry(nextLogIndex); + closeDB(); + try { + RocksDB.destroyDB(this.path, opt); + onReset(nextLogIndex); + if (initAndLoad(null)) { + if (entry == null) { + entry = new LogEntry(); + entry.setType(EntryType.ENTRY_TYPE_NO_OP); + entry.setId(new LogId(nextLogIndex, 0)); + LOG.warn("Entry not found for nextLogIndex {} when reset in data path: {}.", nextLogIndex, this.path); + } + return appendEntry(entry); + } else { + return false; + } + } catch (final RocksDBException e) { + LOG.error("Fail to reset next log index.", e); + return false; + } + } finally { + this.writeLock.unlock(); + } + } + + // Hooks for {@link RocksDBSegmentLogStorage} + + /** + * Called after opening RocksDB and loading configuration into conf manager. + */ + protected boolean onInitLoaded() { + return true; + } + + /** + * Called after closing db. + */ + protected void onShutdown() { + } + + /** + * Called after resetting db. + * + * @param nextLogIndex next log index + */ + protected void onReset(final long nextLogIndex) { + } + + /** + * Called after truncating prefix logs in rocksdb. + * + * @param startIndex the start index + * @param firstIndexKept the first index to kept + */ + protected void onTruncatePrefix(final long startIndex, final long firstIndexKept) throws RocksDBException, + IOException { + } + + /** + * Called when sync data into file system. + */ + protected void onSync() throws IOException, InterruptedException { + } + + protected boolean isSync() { + return this.sync; + } + + /** + * Called after truncating suffix logs in rocksdb. + * + * @param lastIndexKept the last index to kept + */ + protected void onTruncateSuffix(final long lastIndexKept) throws RocksDBException, IOException { + } + + protected WriteContext newWriteContext() { + return EmptyWriteContext.INSTANCE; + } + + /** + * Called before appending data entry. + * + * @param logIndex the log index + * @param value the data value in log entry. + * @return the new value + */ + protected byte[] onDataAppend(final long logIndex, final byte[] value, + final WriteContext ctx) throws IOException, InterruptedException { + ctx.finishJob(); + return value; + } + + /** + * Called after getting data from rocksdb. + * + * @param logIndex the log index + * @param value the value in rocksdb + * @return the new value + */ + protected byte[] onDataGet(final long logIndex, final byte[] value) throws IOException { + return value; + } + + @Override + public void describe(final Printer out) { + this.readLock.lock(); + try { + if (this.db != null) { + out.println(this.db.getProperty("rocksdb.stats")); + } + out.println(""); + if (this.statistics != null) { + out.println(this.statistics.getString()); + } + } catch (final RocksDBException e) { + out.println(e); + } finally { + this.readLock.unlock(); + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/FileReader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/FileReader.java new file mode 100644 index 0000000..f67ee96 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/FileReader.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.io; + +import java.io.IOException; + +import com.alipay.sofa.jraft.error.RetryAgainException; +import com.alipay.sofa.jraft.util.ByteBufferCollector; + +/** + * Read data from a file, all the method should be thread-safe. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-13 11:48:34 AM + */ +public interface FileReader { + + int EOF = -1; + + /** + * Get the file path. + * + * @return path of the file + */ + String getPath(); + + /** + * Read file into buf starts from offset at most maxCount. + * + * @param buf read bytes into this buf + * @param fileName file name + * @param offset the offset of file + * @param maxCount max read bytes + * @return -1 if reaches end, else return read count. + * @throws IOException if some I/O error occurs + * @throws RetryAgainException if it's not allowed to read partly + * or it's allowed but throughput is throttled to 0, try again. + */ + int readFile(final ByteBufferCollector buf, final String fileName, final long offset, final long maxCount) + throws IOException, + RetryAgainException; +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/LocalDirReader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/LocalDirReader.java new file mode 100644 index 0000000..affb435 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/LocalDirReader.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.io; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.channels.FileChannel; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.error.RetryAgainException; +import com.alipay.sofa.jraft.util.ByteBufferCollector; +import com.google.protobuf.Message; + +/** + * Read a file data form local dir by fileName. + * + * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Apr-06 9:25:12 PM + */ +public class LocalDirReader implements FileReader { + + private static final Logger LOG = LoggerFactory.getLogger(LocalDirReader.class); + + private final String path; + + public LocalDirReader(String path) { + super(); + this.path = path; + } + + @Override + public String getPath() { + return path; + } + + @Override + public int readFile(final ByteBufferCollector buf, final String fileName, final long offset, final long maxCount) + throws IOException, + RetryAgainException { + return readFileWithMeta(buf, fileName, null, offset, maxCount); + } + + @SuppressWarnings("unused") + protected int readFileWithMeta(final ByteBufferCollector buf, final String fileName, final Message fileMeta, + long offset, final long maxCount) throws IOException, RetryAgainException { + buf.expandIfNecessary(); + final String filePath = this.path + File.separator + fileName; + final File file = new File(filePath); + try (final FileInputStream input = new FileInputStream(file); final FileChannel fc = input.getChannel()) { + int totalRead = 0; + while (true) { + final int nread = fc.read(buf.getBuffer(), offset); + if (nread <= 0) { + return EOF; + } + totalRead += nread; + if (totalRead < maxCount) { + if (buf.hasRemaining()) { + return EOF; + } else { + buf.expandAtMost((int) (maxCount - totalRead)); + offset += nread; + } + } else { + final long fsize = file.length(); + if (fsize < 0) { + LOG.warn("Invalid file length {}", filePath); + return EOF; + } + if (fsize == offset + nread) { + return EOF; + } else { + return totalRead; + } + } + } + } + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java new file mode 100644 index 0000000..2e48f15 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/io/ProtoBufFile.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.io; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import com.alipay.sofa.jraft.rpc.ProtobufMsgFactory; +import com.alipay.sofa.jraft.util.Bits; +import com.alipay.sofa.jraft.util.Utils; +import com.google.protobuf.Message; + +/** + * A file to store protobuf message. Format: + *
    + *
  • class name length(4 bytes)
    • + *
  • class name
    • + *
  • msg length(4 bytes)
    • + *
  • msg data
    • + *
+ * @author boyan (boyan@alibaba-inc.com) + * + * 2018-Mar-12 8:56:23 PM + */ +public class ProtoBufFile { + + static { + ProtobufMsgFactory.load(); + } + + /** file path */ + private final String path; + + public ProtoBufFile(final String path) { + this.path = path; + } + + /** + * Load a protobuf message from file. + */ + public T load() throws IOException { + File file = new File(this.path); + + if (!file.exists()) { + return null; + } + + final byte[] lenBytes = new byte[4]; + try (final FileInputStream fin = new FileInputStream(file); + final BufferedInputStream input = new BufferedInputStream(fin)) { + readBytes(lenBytes, input); + final int len = Bits.getInt(lenBytes, 0); + if (len <= 0) { + throw new IOException("Invalid message fullName."); + } + final byte[] nameBytes = new byte[len]; + readBytes(nameBytes, input); + final String name = new String(nameBytes); + readBytes(lenBytes, input); + final int msgLen = Bits.getInt(lenBytes, 0); + final byte[] msgBytes = new byte[msgLen]; + readBytes(msgBytes, input); + return ProtobufMsgFactory.newMessageByProtoClassName(name, msgBytes); + } + } + + private void readBytes(final byte[] bs, final InputStream input) throws IOException { + int read; + if ((read = input.read(bs)) != bs.length) { + throw new IOException("Read error, expects " + bs.length + " bytes, but read " + read); + } + } + + /** + * Save a protobuf message to file. + * + * @param msg protobuf message + * @param sync if sync flush data to disk + * @return true if save success + */ + public boolean save(final Message msg, final boolean sync) throws IOException { + // Write message into temp file + final File file = new File(this.path + ".tmp"); + try (final FileOutputStream fOut = new FileOutputStream(file); + final BufferedOutputStream output = new BufferedOutputStream(fOut)) { + final byte[] lenBytes = new byte[4]; + + // name len + name + final String fullName = msg.getDescriptorForType().getFullName(); + final int nameLen = fullName.length(); + Bits.putInt(lenBytes, 0, nameLen); + output.write(lenBytes); + output.write(fullName.getBytes()); + // msg len + msg + final int msgLen = msg.getSerializedSize(); + Bits.putInt(lenBytes, 0, msgLen); + output.write(lenBytes); + msg.writeTo(output); + output.flush(); + } + if (sync) { + Utils.fsync(file); + } + + return Utils.atomicMoveFile(file, new File(this.path), sync); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/AbortFile.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/AbortFile.java new file mode 100644 index 0000000..84f5f6c --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/AbortFile.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.log; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Date; + +/** + * Abort file + * + * @author boyan(boyan@antfin.com) + */ +public class AbortFile { + + private final String path; + + public String getPath() { + return this.path; + } + + public AbortFile(final String path) { + super(); + this.path = path; + } + + public boolean create() throws IOException { + final File file = new File(this.path); + if (file.createNewFile()) { + writeDate(); + return true; + } else { + return false; + } + } + + @SuppressWarnings("deprecation") + private void writeDate() throws IOException { + final File file = new File(this.path); + try (final FileWriter writer = new FileWriter(file, false)) { + writer.write(new Date().toGMTString()); + writer.write(System.lineSeparator()); + } + } + + public void touch() throws IOException { + writeDate(); + } + + public boolean exists() { + final File file = new File(this.path); + return file.isFile() && file.exists(); + } + + public boolean destroy() { + return new File(this.path) // + .delete(); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/CheckpointFile.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/CheckpointFile.java new file mode 100644 index 0000000..88a03a9 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/CheckpointFile.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.log; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; + +import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta; +import com.alipay.sofa.jraft.storage.io.ProtoBufFile; +import com.alipay.sofa.jraft.util.AsciiStringUtil; +import com.alipay.sofa.jraft.util.Bits; +import com.google.protobuf.ZeroByteStringHelper; + +/** + * Segments checkpoint file. + * + * @author boyan(boyan@antfin.com) + */ +public class CheckpointFile { + /** + * Checkpoint metadata info. + * + * @author boyan(boyan@antfin.com) + */ + public static final class Checkpoint { + // Segment file name + public String segFilename; + // Segment file current commit position. + public int committedPos; + + public Checkpoint(final String segFilename, final int committedPos) { + super(); + this.segFilename = segFilename; + this.committedPos = committedPos; + } + + /** + * commitPos (4 bytes) + path(4 byte len + string bytes) + */ + byte[] encode() { + byte[] ps = AsciiStringUtil.unsafeEncode(this.segFilename); + byte[] bs = new byte[8 + ps.length]; + Bits.putInt(bs, 0, this.committedPos); + Bits.putInt(bs, 4, ps.length); + System.arraycopy(ps, 0, bs, 8, ps.length); + return bs; + } + + boolean decode(final byte[] bs) { + if (bs.length < 8) { + return false; + } + this.committedPos = Bits.getInt(bs, 0); + int len = Bits.getInt(bs, 4); + this.segFilename = AsciiStringUtil.unsafeDecode(bs, 8, len); + return this.committedPos >= 0 && !this.segFilename.isEmpty(); + } + + @Override + public String toString() { + return "Checkpoint [segFilename=" + this.segFilename + ", committedPos=" + this.committedPos + "]"; + } + } + + public void destroy() { + FileUtils.deleteQuietly(new File(this.path)); + } + + public String getPath() { + return this.path; + } + + private final String path; + + public CheckpointFile(final String path) { + super(); + this.path = path; + } + + public synchronized boolean save(final Checkpoint checkpoint) throws IOException { + final ProtoBufFile file = new ProtoBufFile(this.path); + final byte[] data = checkpoint.encode(); + + final LocalFileMeta meta = LocalFileMeta.newBuilder() // + .setUserMeta(ZeroByteStringHelper.wrap(data)) // + .build(); + + return file.save(meta, true); + } + + public Checkpoint load() throws IOException { + final ProtoBufFile file = new ProtoBufFile(this.path); + final LocalFileMeta meta = file.load(); + if (meta != null) { + final byte[] data = meta.getUserMeta().toByteArray(); + Checkpoint checkpoint = new Checkpoint(null, -1); + if (checkpoint.decode(data)) { + return checkpoint; + } + } + return null; + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/LibC.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/LibC.java new file mode 100644 index 0000000..e68ba32 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/LibC.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.log; + +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.NativeLong; +import com.sun.jna.Platform; +import com.sun.jna.Pointer; + +/** + * Moved from rocketmq. + * + * https://raw.githubusercontent.com/apache/rocketmq/master/store/src/main/java/org/apache/rocketmq/store/util/LibC.java + * @author boyan(boyan@antfin.com) + * + */ +public interface LibC extends Library { + LibC INSTANCE = Native.load(Platform.isWindows() ? "msvcrt" : "c", LibC.class); + + int MADV_WILLNEED = 3; + int MADV_DONTNEED = 4; + + int MCL_CURRENT = 1; + int MCL_FUTURE = 2; + int MCL_ONFAULT = 4; + + /* sync memory asynchronously */ + int MS_ASYNC = 0x0001; + /* invalidate mappings & caches */ + int MS_INVALIDATE = 0x0002; + /* synchronous memory sync */ + int MS_SYNC = 0x0004; + + int mlock(Pointer var1, NativeLong var2); + + int munlock(Pointer var1, NativeLong var2); + + int madvise(Pointer var1, NativeLong var2, int var3); + + Pointer memset(Pointer p, int v, long len); + + int mlockall(int flags); + + int msync(Pointer p, NativeLong length, int flags); +} \ No newline at end of file diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/RocksDBSegmentLogStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/RocksDBSegmentLogStorage.java new file mode 100644 index 0000000..7b87d23 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/RocksDBSegmentLogStorage.java @@ -0,0 +1,1214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.log; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.regex.Pattern; + +import org.apache.commons.io.FileUtils; +import org.rocksdb.RocksDBException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.common.profile.StringUtil; +import com.alipay.sofa.jraft.option.RaftOptions; +import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage; +import com.alipay.sofa.jraft.storage.log.CheckpointFile.Checkpoint; +import com.alipay.sofa.jraft.storage.log.SegmentFile.SegmentFileOptions; +import com.alipay.sofa.jraft.util.ArrayDeque; +import com.alipay.sofa.jraft.util.Bits; +import com.alipay.sofa.jraft.util.CountDownEvent; +import com.alipay.sofa.jraft.util.NamedThreadFactory; +import com.alipay.sofa.jraft.util.Platform; +import com.alipay.sofa.jraft.util.Requires; +import com.alipay.sofa.jraft.util.SystemPropertyUtil; +import com.alipay.sofa.jraft.util.ThreadPoolUtil; +import com.alipay.sofa.jraft.util.Utils; + +/** + * Log Storage implementation based on rocksdb and segment files. + * + * @author boyan(boyan@antfin.com) + */ +public class RocksDBSegmentLogStorage extends RocksDBLogStorage { + + private static final int PRE_ALLOCATE_SEGMENT_COUNT = 2; + private static final int MEM_SEGMENT_COUNT = 3; + + private static class AllocatedResult { + SegmentFile segmentFile; + IOException ie; + + public AllocatedResult(final SegmentFile segmentFile) { + super(); + this.segmentFile = segmentFile; + } + + public AllocatedResult(final IOException ie) { + super(); + this.ie = ie; + } + + } + + public static class BarrierWriteContext implements WriteContext { + private final CountDownEvent events = new CountDownEvent(); + private volatile Exception e; + private volatile List hooks; + + @Override + public void startJob() { + this.events.incrementAndGet(); + } + + @Override + public synchronized void addFinishHook(final Runnable r) { + if (this.hooks == null) { + this.hooks = new CopyOnWriteArrayList<>(); + } + this.hooks.add(r); + } + + @Override + public void finishJob() { + this.events.countDown(); + } + + @Override + public void setError(final Exception e) { + this.e = e; + } + + @Override + public void joinAll() throws InterruptedException, IOException { + this.events.await(); + if (this.hooks != null) { + for (Runnable r : this.hooks) { + r.run(); + } + } + if (this.e != null) { + throw new IOException("Fail to apppend entries", this.e); + } + } + + } + + private static final String SEGMENT_FILE_POSFIX = ".s"; + + private static final Logger LOG = LoggerFactory + .getLogger(RocksDBSegmentLogStorage.class); + + /** + * Default checkpoint interval in milliseconds. + */ + private static final int DEFAULT_CHECKPOINT_INTERVAL_MS = SystemPropertyUtil.getInt( + "jraft.log_storage.segment.checkpoint.interval.ms", + 5000); + + /** + * Location metadata format: + * 1. magic bytes + * 2. reserved(2 B) + * 3. segmentFileName(8 B) + * 4. wrotePosition(4 B) + */ + private static final int LOCATION_METADATA_SIZE = SegmentFile.RECORD_MAGIC_BYTES_SIZE + 2 + 8 + 4; + + /** + * Max segment file size, 1G + */ + private static final int MAX_SEGMENT_FILE_SIZE = SystemPropertyUtil.getInt( + "jraft.log_storage.segment.max.size.bytes", + 1024 * 1024 * 1024); + + // Default value size threshold to decide whether it will be stored in segments or rocksdb, default is 4K. + // When the value size is less than 4K, it will be stored in rocksdb directly. + private static int DEFAULT_VALUE_SIZE_THRESHOLD = SystemPropertyUtil.getInt( + "jraft.log_storage.segment.value.threshold.bytes", + 4 * 1024); + + /** + * RocksDBSegmentLogStorage builder + * @author boyan(boyan@antfin.com) + * + */ + public static class Builder { + private String path; + private RaftOptions raftOptions; + private int valueSizeThreshold = DEFAULT_VALUE_SIZE_THRESHOLD; + private int maxSegmentFileSize = MAX_SEGMENT_FILE_SIZE; + private ThreadPoolExecutor writeExecutor; + private int preAllocateSegmentCount = PRE_ALLOCATE_SEGMENT_COUNT; + private int keepInMemorySegmentCount = MEM_SEGMENT_COUNT; + private int checkpointIntervalMs = DEFAULT_CHECKPOINT_INTERVAL_MS; + + public String getPath() { + return this.path; + } + + public Builder setPath(final String path) { + this.path = path; + return this; + } + + public RaftOptions getRaftOptions() { + return this.raftOptions; + } + + public Builder setRaftOptions(final RaftOptions raftOptions) { + this.raftOptions = raftOptions; + return this; + } + + public int getValueSizeThreshold() { + return this.valueSizeThreshold; + } + + public Builder setValueSizeThreshold(final int valueSizeThreshold) { + this.valueSizeThreshold = valueSizeThreshold; + return this; + } + + public int getMaxSegmentFileSize() { + return this.maxSegmentFileSize; + } + + public Builder setMaxSegmentFileSize(final int maxSegmentFileSize) { + this.maxSegmentFileSize = maxSegmentFileSize; + return this; + } + + public ThreadPoolExecutor getWriteExecutor() { + return this.writeExecutor; + } + + public Builder setWriteExecutor(final ThreadPoolExecutor writeExecutor) { + this.writeExecutor = writeExecutor; + return this; + } + + public int getPreAllocateSegmentCount() { + return this.preAllocateSegmentCount; + } + + public Builder setPreAllocateSegmentCount(final int preAllocateSegmentCount) { + this.preAllocateSegmentCount = preAllocateSegmentCount; + return this; + } + + public int getKeepInMemorySegmentCount() { + return this.keepInMemorySegmentCount; + } + + public Builder setKeepInMemorySegmentCount(final int keepInMemorySegmentCount) { + this.keepInMemorySegmentCount = keepInMemorySegmentCount; + return this; + } + + public int getCheckpointIntervalMs() { + return this.checkpointIntervalMs; + } + + public Builder setCheckpointIntervalMs(final int checkpointIntervalMs) { + this.checkpointIntervalMs = checkpointIntervalMs; + return this; + } + + public RocksDBSegmentLogStorage build() { + return new RocksDBSegmentLogStorage(this.path, this.raftOptions, this.valueSizeThreshold, + this.maxSegmentFileSize, this.preAllocateSegmentCount, this.keepInMemorySegmentCount, + this.checkpointIntervalMs, this.writeExecutor); + } + + } + + private final int valueSizeThreshold; + private final String segmentsPath; + private final CheckpointFile checkpointFile; + // used or using segments. + private List segments; + // pre-allocated and blank segments. + private ArrayDeque blankSegments; + private final Lock allocateLock = new ReentrantLock(); + private final Condition fullCond = this.allocateLock.newCondition(); + private final Condition emptyCond = this.allocateLock.newCondition(); + // segment file sequence. + private final AtomicLong nextFileSequence = new AtomicLong(0); + private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final Lock writeLock = this.readWriteLock.writeLock(); + private final Lock readLock = this.readWriteLock.readLock(); + private ScheduledExecutorService checkpointExecutor; + private final AbortFile abortFile; + private final ThreadPoolExecutor writeExecutor; + private Thread segmentAllocator; + private final int maxSegmentFileSize; + private int preAllocateSegmentCount = PRE_ALLOCATE_SEGMENT_COUNT; + private int keepInMemorySegmentCount = MEM_SEGMENT_COUNT; + private int checkpointIntervalMs = DEFAULT_CHECKPOINT_INTERVAL_MS; + + /** + * Creates a RocksDBSegmentLogStorage builder. + * @return a builder instance. + */ + public static final Builder builder(final String uri, final RaftOptions raftOptions) { + return new Builder().setPath(uri).setRaftOptions(raftOptions); + } + + public RocksDBSegmentLogStorage(final String path, final RaftOptions raftOptions) { + this(path, raftOptions, DEFAULT_VALUE_SIZE_THRESHOLD, MAX_SEGMENT_FILE_SIZE); + } + + public RocksDBSegmentLogStorage(final String path, final RaftOptions raftOptions, final int valueSizeThreshold, + final int maxSegmentFileSize) { + this(path, raftOptions, valueSizeThreshold, maxSegmentFileSize, PRE_ALLOCATE_SEGMENT_COUNT, MEM_SEGMENT_COUNT, + DEFAULT_CHECKPOINT_INTERVAL_MS, createDefaultWriteExecutor()); + } + + private static ThreadPoolExecutor createDefaultWriteExecutor() { + return ThreadPoolUtil.newThreadPool("RocksDBSegmentLogStorage-write-pool", true, Utils.cpus(), + Utils.cpus() * 3, 60, new ArrayBlockingQueue<>(10000), new NamedThreadFactory( + "RocksDBSegmentLogStorageWriter"), new ThreadPoolExecutor.CallerRunsPolicy()); + } + + public RocksDBSegmentLogStorage(final String path, final RaftOptions raftOptions, final int valueSizeThreshold, + final int maxSegmentFileSize, final int preAllocateSegmentCount, + final int keepInMemorySegmentCount, final int checkpointIntervalMs, + final ThreadPoolExecutor writeExecutor) { + super(path, raftOptions); + if (Platform.isMac()) { + LOG.warn("RocksDBSegmentLogStorage is not recommended on mac os x, it's performance is poorer than RocksDBLogStorage."); + } + Requires.requireTrue(maxSegmentFileSize > 0, "maxSegmentFileSize is not greater than zero"); + Requires.requireTrue(preAllocateSegmentCount > 0, "preAllocateSegmentCount is not greater than zero"); + Requires.requireTrue(checkpointIntervalMs > 0, "checkpointIntervalMs is not greater than zero"); + Requires.requireTrue(keepInMemorySegmentCount > 0, "keepInMemorySegmentCount is not greater than zero"); + this.segmentsPath = path + File.separator + "segments"; + this.abortFile = new AbortFile(this.segmentsPath + File.separator + "abort"); + this.checkpointFile = new CheckpointFile(this.segmentsPath + File.separator + "checkpoint"); + this.valueSizeThreshold = valueSizeThreshold; + this.maxSegmentFileSize = maxSegmentFileSize; + this.writeExecutor = writeExecutor == null ? createDefaultWriteExecutor() : writeExecutor; + this.preAllocateSegmentCount = preAllocateSegmentCount; + this.checkpointIntervalMs = checkpointIntervalMs; + this.keepInMemorySegmentCount = keepInMemorySegmentCount; + + } + + private SegmentFile getLastSegmentFile(final long logIndex, final int waitToWroteSize, + final boolean createIfNecessary, final WriteContext ctx) throws IOException, + InterruptedException { + SegmentFile lastFile = null; + while (true) { + int segmentCount = 0; + this.readLock.lock(); + try { + + if (!this.segments.isEmpty()) { + segmentCount = this.segments.size(); + final SegmentFile currLastFile = getLastSegmentWithoutLock(); + if (waitToWroteSize <= 0 || !currLastFile.reachesFileEndBy(waitToWroteSize)) { + lastFile = currLastFile; + } + } + } finally { + this.readLock.unlock(); + } + if (lastFile == null && createIfNecessary) { + lastFile = createNewSegmentFile(logIndex, segmentCount, ctx); + if (lastFile != null) { + return lastFile; + } else { + // Try again + continue; + } + } + return lastFile; + } + } + + private SegmentFile createNewSegmentFile(final long logIndex, final int oldSegmentCount, + final WriteContext ctx) throws InterruptedException, IOException { + SegmentFile segmentFile = null; + this.writeLock.lock(); + try { + // CAS by segments count. + if (this.segments.size() != oldSegmentCount) { + return segmentFile; + } + if (!this.segments.isEmpty()) { + // Sync current last file and correct it's lastLogIndex. + final SegmentFile currLastFile = getLastSegmentWithoutLock(); + currLastFile.setLastLogIndex(logIndex - 1); + ctx.startJob(); + // Attach a finish hook to set last segment file to be read-only. + ctx.addFinishHook(() -> currLastFile.setReadOnly(true)); + // Run sync in parallel + this.writeExecutor.execute(() -> { + try { + currLastFile.sync(isSync()); + } catch (final IOException e) { + ctx.setError(e); + } finally { + ctx.finishJob(); + } + + }); + + } + segmentFile = allocateSegmentFile(logIndex); + return segmentFile; + } finally { + this.writeLock.unlock(); + } + + } + + private SegmentFile allocateSegmentFile(final long index) throws InterruptedException, IOException { + this.allocateLock.lock(); + try { + while (this.blankSegments.isEmpty()) { + this.emptyCond.await(); + } + final AllocatedResult result = this.blankSegments.pollFirst(); + if (result.ie != null) { + throw result.ie; + } + this.fullCond.signal(); + result.segmentFile.setFirstLogIndex(index); + this.segments.add(result.segmentFile); + return result.segmentFile; + } finally { + this.allocateLock.unlock(); + } + } + + private SegmentFile allocateNewSegmentFile() throws IOException { + final String newSegPath = getNewSegmentFilePath(); + SegmentFile segmentFile = new SegmentFile(this.maxSegmentFileSize, newSegPath, this.writeExecutor); + final SegmentFileOptions opts = SegmentFileOptions.builder() // + .setSync(false) // + .setRecover(false) // + .setLastFile(true) // + .setNewFile(true) // + .setPos(0).build(); + + try { + if (!segmentFile.init(opts)) { + throw new IOException("Fail to create new segment file"); + } + segmentFile.hintLoad(); + LOG.info("Create a new segment file {}.", segmentFile.getPath()); + return segmentFile; + } catch (IOException e) { + // Delete the file if fails + FileUtils.deleteQuietly(new File(newSegPath)); + throw e; + } + } + + private String getNewSegmentFilePath() { + return this.segmentsPath + File.separator + String.format("%019d", this.nextFileSequence.getAndIncrement()) + + SEGMENT_FILE_POSFIX; + } + + @Override + protected void onSync() throws IOException, InterruptedException { + final SegmentFile lastSegmentFile = getLastSegmentFileForRead(); + if (lastSegmentFile != null) { + lastSegmentFile.sync(isSync()); + } + } + + private static final Pattern SEGMENT_FILE_NAME_PATTERN = Pattern.compile("[0-9]+\\.s"); + + @Override + protected boolean onInitLoaded() { + final long startMs = Utils.monotonicMs(); + this.writeLock.lock(); + try { + final File segmentsDir = new File(this.segmentsPath); + if (!ensureDir(segmentsDir)) { + return false; + } + final Checkpoint checkpoint = loadCheckpoint(); + + final File[] segmentFiles = segmentsDir + .listFiles((final File dir, final String name) -> SEGMENT_FILE_NAME_PATTERN.matcher(name).matches()); + + final boolean normalExit = !this.abortFile.exists(); + if (!normalExit) { + LOG.info("{} {} did not exit normally, will try to recover last file.", getServiceName(), + this.segmentsPath); + } + this.segments = new ArrayList<>(segmentFiles == null ? 10 : segmentFiles.length); + this.blankSegments = new ArrayDeque<>(); + List corruptedHeaderSegments = new ArrayList<>(); + + if (segmentFiles != null && segmentFiles.length > 0) { + // Sort by sequences. + Arrays.sort(segmentFiles, Comparator.comparing(RocksDBSegmentLogStorage::getFileSequenceFromFileName)); + + final String checkpointSegFile = getCheckpointSegFilePath(checkpoint); + + // mmap files + for (int i = 0; i < segmentFiles.length; i++) { + final File segFile = segmentFiles[i]; + this.nextFileSequence.set(getFileSequenceFromFileName(segFile) + 1); + final SegmentFile segmentFile = new SegmentFile(this.maxSegmentFileSize, segFile.getAbsolutePath(), + this.writeExecutor); + + if (!segmentFile.mmapFile(false)) { + assert (segmentFile.isHeaderCorrupted()); + corruptedHeaderSegments.add(segFile); + continue; + } + + if (segmentFile.isBlank()) { + this.blankSegments.add(new AllocatedResult(segmentFile)); + } else if (segmentFile.isHeaderCorrupted()) { + corruptedHeaderSegments.add(segFile); + } else { + this.segments.add(segmentFile); + } + } + + // Processing corrupted header files + //TODO(boyan) maybe we can find a better solution for such case that new allocated segment file is corrupted when power failure etc. + if(!processCorruptedHeaderFiles(corruptedHeaderSegments)) { + return false; + } + + // init blank segments + if(!initBlankFiles()) { + return false; + } + + // try to recover segments + if(!recoverFiles(checkpoint, normalExit, checkpointSegFile)) { + return false; + } + } else { + if (checkpoint != null) { + LOG.warn("Missing segment files, checkpoint is: {}", checkpoint); + return false; + } + } + + LOG.info("{} Loaded {} segment files and {} blank segment files from path {}.", getServiceName(), + this.segments.size(), this.blankSegments.size(), this.segmentsPath); + + LOG.info("{} segments: \n{}", getServiceName(), descSegments()); + + startCheckpointTask(); + + if (normalExit) { + if (!this.abortFile.create()) { + LOG.error("Fail to create abort file {}.", this.abortFile.getPath()); + return false; + } + } else { + this.abortFile.touch(); + } + startSegmentAllocator(); + + return true; + } catch (final Exception e) { + LOG.error("Fail to load segment files from directory {}.", this.segmentsPath, e); + return false; + } finally { + this.writeLock.unlock(); + LOG.info("{} init and load cost {} ms.", getServiceName(), Utils.monotonicMs() - startMs); + } + } + + private boolean recoverFiles(final Checkpoint checkpoint, final boolean normalExit, final String checkpointSegFile) { + boolean needRecover = false; + SegmentFile prevFile = null; + for (int i = 0; i < this.segments.size(); i++) { + final boolean isLastFile = i == this.segments.size() - 1; + SegmentFile segmentFile = this.segments.get(i); + int pos = segmentFile.getSize(); + if (StringUtil.equalsIgnoreCase(checkpointSegFile, segmentFile.getFilename())) { + needRecover = true; + assert (checkpoint != null); + pos = checkpoint.committedPos; + } else { + if (needRecover) { + pos = 0; + } + } + + final SegmentFileOptions opts = SegmentFileOptions.builder() // + .setSync(isSync()) // + .setRecover(needRecover && !normalExit) // + .setLastFile(isLastFile) // + .setNewFile(false) // + .setPos(pos).build(); + + if (!segmentFile.init(opts)) { + LOG.error("Fail to load segment file {}.", segmentFile.getPath()); + segmentFile.shutdown(); + return false; + } + /** + * It's wrote position is from start(HEADER_SIZE) but it's not the last file, SHOULD not happen. + */ + if (segmentFile.getWrotePos() == SegmentFile.HEADER_SIZE && !isLastFile) { + LOG.error("Detected corrupted segment file {}.", segmentFile.getPath()); + return false; + } + + if (prevFile != null) { + prevFile.setLastLogIndex(segmentFile.getFirstLogIndex() - 1); + } + prevFile = segmentFile; + } + if (getLastLogIndex() > 0 && prevFile != null) { + prevFile.setLastLogIndex(getLastLogIndex()); + } + return true; + } + + private boolean initBlankFiles() { + for (AllocatedResult ret : this.blankSegments) { + final SegmentFile segmentFile = ret.segmentFile; + final SegmentFileOptions opts = SegmentFileOptions.builder() // + .setSync(false) // + .setRecover(false) // + .setLastFile(true) // + .build(); + + if (!segmentFile.init(opts)) { + LOG.error("Fail to load blank segment file {}.", segmentFile.getPath()); + segmentFile.shutdown(); + return false; + } + } + return true; + } + + private boolean processCorruptedHeaderFiles(final List corruptedHeaderSegments) throws IOException { + if (corruptedHeaderSegments.size() == 1) { + final File corruptedFile = corruptedHeaderSegments.get(0); + if (getFileSequenceFromFileName(corruptedFile) != this.nextFileSequence.get() - 1) { + LOG.error("Detected corrupted header segment file {}.", corruptedFile); + return false; + } else { + // The file is the last file,it's the new blank segment but fail to save header, we can + // remove it safely. + LOG.warn("Truncate the last segment file {} which it's header is corrupted.", + corruptedFile.getAbsolutePath()); + // We don't want to delete it, but rename it for safety. + FileUtils.moveFile(corruptedFile, new File(corruptedFile.getAbsolutePath() + ".corrupted")); + } + } else if (corruptedHeaderSegments.size() > 1) { + // FATAL: it should not happen. + LOG.error("Detected corrupted header segment files: {}.", corruptedHeaderSegments); + return false; + } + + return true; + } + + private void startSegmentAllocator() throws IOException { + // Warmup + if (this.blankSegments.isEmpty()) { + doAllocateSegment0(); + } + // Start the thread. + this.segmentAllocator = new Thread(this::doAllocateSegment); + this.segmentAllocator.setDaemon(true); + this.segmentAllocator.setName("SegmentAllocator"); + this.segmentAllocator.start(); + } + + private void doAllocateSegment() { + LOG.info("SegmentAllocator is started."); + while (!Thread.currentThread().isInterrupted()) { + doAllocateSegmentInLock(); + doSwapOutSegments(false); + } + LOG.info("SegmentAllocator exit."); + } + + private void doAllocateSegmentInLock() { + this.allocateLock.lock(); + try { + //TODO configure cap + while (this.blankSegments.size() >= this.preAllocateSegmentCount) { + this.fullCond.await(); + } + doAllocateSegment0(); + this.emptyCond.signal(); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final IOException e) { + this.blankSegments.add(new AllocatedResult(e)); + this.emptyCond.signal(); + } finally { + this.allocateLock.unlock(); + } + } + + private void doSwapOutSegments(final boolean force) { + if (force) { + this.readLock.lock(); + } else if (!this.readLock.tryLock()) { + return; + } + try { + if (this.segments.size() <= this.keepInMemorySegmentCount) { + return; + } + int segmentsInMemeCount = 0; + int swappedOutCount = 0; + final long beginTime = Utils.monotonicMs(); + final int lastIndex = this.segments.size() - 1; + for (int i = lastIndex; i >= 0; i--) { + SegmentFile segFile = this.segments.get(i); + if (!segFile.isSwappedOut()) { + segmentsInMemeCount++; + if (segmentsInMemeCount >= this.keepInMemorySegmentCount && i != lastIndex) { + segFile.hintUnload(); + segFile.swapOut(); + swappedOutCount++; + } + } + } + LOG.info("Swapped out {} segment files, cost {} ms.", swappedOutCount, Utils.monotonicMs() - beginTime); + } catch (final Exception e) { + LOG.error("Fail to swap out segments.", e); + } finally { + this.readLock.unlock(); + } + } + + private void doAllocateSegment0() throws IOException { + SegmentFile segFile = allocateNewSegmentFile(); + this.blankSegments.add(new AllocatedResult(segFile)); + } + + private static long getFileSequenceFromFileName(final File file) { + final String name = file.getName(); + assert (name.endsWith(SEGMENT_FILE_POSFIX)); + int idx = name.indexOf(SEGMENT_FILE_POSFIX); + return Long.valueOf(name.substring(0, idx)); + } + + private Checkpoint loadCheckpoint() { + final Checkpoint checkpoint; + try { + checkpoint = this.checkpointFile.load(); + if (checkpoint != null) { + LOG.info("Loaded checkpoint: {} from {}.", checkpoint, this.checkpointFile.getPath()); + } + } catch (final IOException e) { + LOG.error("Fail to load checkpoint file: {}", this.checkpointFile.getPath(), e); + return null; + } + return checkpoint; + } + + private boolean ensureDir(final File segmentsDir) { + try { + FileUtils.forceMkdir(segmentsDir); + return true; + } catch (final IOException e) { + LOG.error("Fail to create segments directory: {}", this.segmentsPath, e); + return false; + } + } + + private String getCheckpointSegFilePath(final Checkpoint checkpoint) { + return checkpoint != null ? checkpoint.segFilename : null; + } + + private void startCheckpointTask() { + this.checkpointExecutor = Executors + .newSingleThreadScheduledExecutor(new NamedThreadFactory(getServiceName() + "-Checkpoint-Thread-", true)); + this.checkpointExecutor.scheduleAtFixedRate(this::doCheckpoint, this.checkpointIntervalMs, + this.checkpointIntervalMs, TimeUnit.MILLISECONDS); + LOG.info("{} started checkpoint task.", getServiceName()); + } + + private StringBuilder descSegments() { + final StringBuilder segmentsDesc = new StringBuilder("[\n"); + for (final SegmentFile segFile : this.segments) { + segmentsDesc.append(" ").append(segFile.toString()).append("\n"); + } + segmentsDesc.append("]"); + return segmentsDesc; + } + + private String getServiceName() { + return this.getClass().getSimpleName(); + } + + private void stopSegmentAllocator() { + this.segmentAllocator.interrupt(); + try { + this.segmentAllocator.join(500); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + @Override + protected void onShutdown() { + stopCheckpointTask(); + stopSegmentAllocator(); + + List shutdownFiles = Collections.emptyList(); + this.writeLock.lock(); + try { + doCheckpoint(); + shutdownFiles = new ArrayList<>(this.segments); + this.segments.clear(); + if (!this.abortFile.destroy()) { + LOG.error("Fail to delete abort file {}.", this.abortFile.getPath()); + } + } finally { + this.writeLock.unlock(); + for (final SegmentFile segmentFile : shutdownFiles) { + segmentFile.shutdown(); + } + shutdownBlankSegments(); + } + this.writeExecutor.shutdown(); + } + + private void shutdownBlankSegments() { + this.allocateLock.lock(); + try { + for (final AllocatedResult ret : this.blankSegments) { + if (ret.segmentFile != null) { + ret.segmentFile.shutdown(); + } + } + } finally { + this.allocateLock.unlock(); + } + } + + private void stopCheckpointTask() { + if (this.checkpointExecutor != null) { + this.checkpointExecutor.shutdownNow(); + try { + this.checkpointExecutor.awaitTermination(10, TimeUnit.SECONDS); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + LOG.info("{} stopped checkpoint task.", getServiceName()); + } + } + + private void doCheckpoint() { + SegmentFile lastSegmentFile = null; + try { + lastSegmentFile = getLastSegmentFileForRead(); + if (lastSegmentFile != null) { + this.checkpointFile.save(new Checkpoint(lastSegmentFile.getFilename(), lastSegmentFile + .getCommittedPos())); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (final IOException e) { + LOG.error("Fatal error, fail to do checkpoint, last segment file is {}.", + lastSegmentFile != null ? lastSegmentFile.getPath() : "null", e); + } + } + + public SegmentFile getLastSegmentFileForRead() throws IOException, InterruptedException { + return getLastSegmentFile(-1, 0, false, null); + } + + @Override + protected void onReset(final long nextLogIndex) { + List destroyedFiles = new ArrayList<>(); + this.writeLock.lock(); + try { + this.checkpointFile.destroy(); + destroyedFiles.addAll(this.segments); + this.segments.clear(); + LOG.info("Destroyed segments and checkpoint in path {} by resetting.", this.segmentsPath); + } finally { + this.writeLock.unlock(); + for (SegmentFile segFile : destroyedFiles) { + segFile.destroy(); + } + } + } + + private SegmentFile getLastSegmentWithoutLock() { + return this.segments.get(this.segments.size() - 1); + } + + @Override + protected void onTruncatePrefix(final long startIndex, final long firstIndexKept) throws RocksDBException, + IOException { + List destroyedFiles = null; + this.writeLock.lock(); + try { + int fromIndex = binarySearchFileIndexByLogIndex(startIndex); + int toIndex = binarySearchFileIndexByLogIndex(firstIndexKept); + + if (fromIndex < 0) { + fromIndex = 0; + } + if (toIndex < 0) { + // When all the segments contain logs that index is smaller than firstIndexKept, + // truncate all segments. + do { + if (!this.segments.isEmpty()) { + if (getLastSegmentWithoutLock().getLastLogIndex() < firstIndexKept) { + toIndex = this.segments.size(); + break; + } + } + LOG.warn("Segment file not found by logIndex={} to be truncate_prefix, current segments:\n{}.", + firstIndexKept, descSegments()); + return; + } while (false); + } + + final List removedFiles = this.segments.subList(fromIndex, toIndex); + destroyedFiles = new ArrayList<>(removedFiles); + removedFiles.clear(); + doCheckpoint(); + } finally { + this.writeLock.unlock(); + if (destroyedFiles != null) { + for (final SegmentFile segmentFile : destroyedFiles) { + segmentFile.destroy(); + } + } + } + } + + private boolean isMetadata(final byte[] data) { + for (int offset = 0; offset < SegmentFile.RECORD_MAGIC_BYTES_SIZE; offset++) { + if (data[offset] != SegmentFile.RECORD_MAGIC_BYTES[offset]) { + return false; + } + } + return true; + } + + private SegmentFile getFirstSegmentWithoutLock() { + return this.segments.get(0); + } + + @Override + protected void onTruncateSuffix(final long lastIndexKept) throws RocksDBException, IOException { + List destroyedFiles = null; + this.writeLock.lock(); + try { + final int keptFileIndex = binarySearchFileIndexByLogIndex(lastIndexKept); + int toIndex = binarySearchFileIndexByLogIndex(getLastLogIndex()); + + if (keptFileIndex < 0) { + // When all the segments contain logs that index is greater than lastIndexKept, + // truncate all segments. + if (!this.segments.isEmpty()) { + final long firstLogIndex = getFirstSegmentWithoutLock().getFirstLogIndex(); + if (firstLogIndex > lastIndexKept) { + final List removedFiles = this.segments.subList(0, this.segments.size()); + destroyedFiles = new ArrayList<>(removedFiles); + removedFiles.clear(); + } + LOG.info( + "Truncating all segments in {} because the first log index {} is greater than lastIndexKept={}", + this.segmentsPath, firstLogIndex, lastIndexKept); + } + + LOG.warn("Segment file not found by logIndex={} to be truncate_suffix, current segments:\n{}.", + lastIndexKept, descSegments()); + return; + } + + if (toIndex < 0) { + toIndex = this.segments.size() - 1; + } + + // Destroyed files after keptFile + final List removedFiles = this.segments.subList(keptFileIndex + 1, toIndex + 1); + destroyedFiles = new ArrayList<>(removedFiles); + removedFiles.clear(); + + // Process logs in keptFile(firstLogIndex=lastIndexKept) + final SegmentFile keptFile = this.segments.get(keptFileIndex); + if (keptFile.isBlank()) { + return; + } + int logWrotePos = -1; // The truncate position in keptFile. + + // Try to find the right position to be truncated. + { + // First, find in right [lastIndexKept + 1, getLastLogIndex()] + long nextIndex = lastIndexKept + 1; + final long endIndex = Math.min(getLastLogIndex(), keptFile.getLastLogIndex()); + while (nextIndex <= endIndex) { + final byte[] data = getValueFromRocksDB(getKeyBytes(nextIndex)); + if (data != null) { + if (data.length == LOCATION_METADATA_SIZE) { + if (!isMetadata(data)) { + // Stored in rocksdb directly. + nextIndex++; + continue; + } + logWrotePos = getWrotePosition(data); + break; + } else { + // Stored in rocksdb directly. + nextIndex++; + } + } else { + // No more data. + break; + } + } + } + + // Not found in [lastIndexKept + 1, getLastLogIndex()] + if (logWrotePos < 0) { + // Second, try to find in left [firstLogIndex, lastIndexKept) when lastIndexKept is not stored in segments. + final byte[] keptData = getValueFromRocksDB(getKeyBytes(lastIndexKept)); + // The kept log's data is not stored in segments. + if (!isMetadata(keptData)) { + //lastIndexKept's log is stored in rocksdb directly, try to find the first previous log that stored in segment. + long prevIndex = lastIndexKept - 1; + final long startIndex = keptFile.getFirstLogIndex(); + while (prevIndex >= startIndex) { + final byte[] data = getValueFromRocksDB(getKeyBytes(prevIndex)); + if (data != null) { + if (data.length == LOCATION_METADATA_SIZE) { + if (!isMetadata(data)) { + // Stored in rocksdb directly. + prevIndex--; + continue; + } + // Found the position. + logWrotePos = getWrotePosition(data); + final byte[] logData = onDataGet(prevIndex, data); + // Skip this log, it should be kept(logs that are less than lastIndexKept should be kept). + // Fine the next log position. + logWrotePos += SegmentFile.getWriteBytes(logData); + break; + } else { + // Stored in rocksdb directly. + prevIndex--; + } + } else { + LOG.warn( + "Log entry not found at index={} when truncating logs suffix from lastIndexKept={}.", + prevIndex, lastIndexKept); + prevIndex--; + } + } + } + } + + if (logWrotePos >= 0 && logWrotePos < keptFile.getSize()) { + // Truncate the file from wrotePos and set it's lastLogIndex=lastIndexKept. + keptFile.truncateSuffix(logWrotePos, lastIndexKept, isSync()); + } + // Finally, do checkpoint. + doCheckpoint(); + + } finally { + this.writeLock.unlock(); + if (destroyedFiles != null) { + for (final SegmentFile segmentFile : destroyedFiles) { + segmentFile.destroy(); + } + } + } + } + + /** + * Retrieve the log wrote position from metadata. + * + * @param data the metadata + * @return the log wrote position + */ + private int getWrotePosition(final byte[] data) { + return Bits.getInt(data, SegmentFile.RECORD_MAGIC_BYTES_SIZE + 2 + 8); + } + + @Override + protected WriteContext newWriteContext() { + return new BarrierWriteContext(); + } + + @Override + protected byte[] onDataAppend(final long logIndex, final byte[] value, final WriteContext ctx) throws IOException, + InterruptedException { + final int waitToWroteBytes = SegmentFile.getWriteBytes(value); + SegmentFile lastSegmentFile = getLastSegmentFile(logIndex, waitToWroteBytes, true, ctx); + if (lastSegmentFile.reachesFileEndBy(waitToWroteBytes)) { + throw new IOException("Too large value size: " + value.length + ", maxSegmentFileSize=" + + this.maxSegmentFileSize); + } + if (value.length < this.valueSizeThreshold) { + // Small value will be stored in rocksdb directly. + lastSegmentFile.setLastLogIndex(logIndex); + ctx.finishJob(); + return value; + } + // Large value is stored in segment file and returns an encoded location info that will be stored in rocksdb. + final int pos = lastSegmentFile.write(logIndex, value, ctx); + final long firstLogIndex = lastSegmentFile.getFirstLogIndex(); + return encodeLocationMetadata(firstLogIndex, pos); + } + + /** + * Encode segment file firstLogIndex(fileName) and position to a byte array in the format of: + *
    + *
  • magic bytes(2 B)
    • + *
  • reserved (2 B)
    • + *
  • firstLogIndex(8 B)
    • + *
  • wrote position(4 B)
    • + *
+ * @param firstLogIndex the first log index + * @param pos the wrote position + * @return segment info + */ + private byte[] encodeLocationMetadata(final long firstLogIndex, final int pos) { + final byte[] newData = new byte[LOCATION_METADATA_SIZE]; + System.arraycopy(SegmentFile.RECORD_MAGIC_BYTES, 0, newData, 0, SegmentFile.RECORD_MAGIC_BYTES_SIZE); + // 2 bytes reserved + Bits.putLong(newData, SegmentFile.RECORD_MAGIC_BYTES_SIZE + 2, firstLogIndex); + Bits.putInt(newData, SegmentFile.RECORD_MAGIC_BYTES_SIZE + 2 + 8, pos); + return newData; + } + + private int binarySearchFileIndexByLogIndex(final long logIndex) { + this.readLock.lock(); + try { + if (this.segments.isEmpty()) { + return -1; + } + if (this.segments.size() == 1) { + final SegmentFile firstFile = this.segments.get(0); + if (firstFile.contains(logIndex)) { + return 0; + } else { + return -1; + } + } + + int low = 0; + int high = this.segments.size() - 1; + + while (low <= high) { + final int mid = (low + high) >>> 1; + + final SegmentFile file = this.segments.get(mid); + if (file.getLastLogIndex() < logIndex) { + low = mid + 1; + } else if (file.getFirstLogIndex() > logIndex) { + high = mid - 1; + } else { + return mid; + } + } + return -(low + 1); + } finally { + this.readLock.unlock(); + } + } + + private SegmentFile binarySearchFileByFirstLogIndex(final long logIndex) { + this.readLock.lock(); + try { + if (this.segments.isEmpty()) { + return null; + } + if (this.segments.size() == 1) { + final SegmentFile firstFile = this.segments.get(0); + if (firstFile.getFirstLogIndex() == logIndex) { + return firstFile; + } else { + return null; + } + } + + int low = 0; + int high = this.segments.size() - 1; + + while (low <= high) { + final int mid = (low + high) >>> 1; + + final SegmentFile file = this.segments.get(mid); + if (file.getFirstLogIndex() < logIndex) { + low = mid + 1; + } else if (file.getFirstLogIndex() > logIndex) { + high = mid - 1; + } else { + return file; + } + } + return null; + } finally { + this.readLock.unlock(); + } + } + + @Override + protected byte[] onDataGet(final long logIndex, final byte[] value) throws IOException { + if (value == null || value.length != LOCATION_METADATA_SIZE) { + return value; + } + + int offset = 0; + for (; offset < SegmentFile.RECORD_MAGIC_BYTES_SIZE; offset++) { + if (value[offset] != SegmentFile.RECORD_MAGIC_BYTES[offset]) { + return value; + } + } + + // skip reserved + offset += 2; + + final long firstLogIndex = Bits.getLong(value, offset); + final int pos = Bits.getInt(value, offset + 8); + final SegmentFile file = binarySearchFileByFirstLogIndex(firstLogIndex); + if (file == null) { + return null; + } + return file.read(logIndex, pos); + } +} diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/SegmentFile.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/SegmentFile.java new file mode 100644 index 0000000..60d7fa5 --- /dev/null +++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/log/SegmentFile.java @@ -0,0 +1,902 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.alipay.sofa.jraft.storage.log; + +import java.io.File; +import java.io.IOException; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.reflect.Method; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.alipay.sofa.jraft.Lifecycle; +import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage.WriteContext; +import com.alipay.sofa.jraft.storage.log.SegmentFile.SegmentFileOptions; +import com.alipay.sofa.jraft.util.Bits; +import com.alipay.sofa.jraft.util.BytesUtil; +import com.alipay.sofa.jraft.util.OnlyForTest; +import com.alipay.sofa.jraft.util.Utils; +import com.sun.jna.NativeLong; +import com.sun.jna.Pointer; + +/** + * A fixed size file. The content format is: + * 
+ *   magic bytes       first log index    reserved
+ *   [0x20 0x20]      [... 8 bytes...]   [8 bytes]
+ *
+ *   [record, record, ...]
+ * 
+ *
+ * Every record format is:
+ * 
+ *   Magic bytes     data length   data
+ *   [0x57, 0x8A]    [4 bytes]     [bytes]
+ *

+ *
+ * @author boyan(boyan@antfin.com)
+ * @since 1.2.6
+ */
+public class SegmentFile implements Lifecycle {
+
+    private static final int  FSYNC_COST_MS_THRESHOLD = 1000;
+    private static final int  ONE_MINUTE              = 60 * 1000;
+    public static final int   HEADER_SIZE             = 18;
+    private static final long BLANK_LOG_INDEX         = -99;
+
+    /**
+     * Segment file header.
+     * @author boyan(boyan@antfin.com)
+     *
+     */
+    public static class SegmentHeader {
+
+        private static final long RESERVED_FLAG = 0L;
+        // The file first log index(inclusive)
+        volatile long             firstLogIndex = BLANK_LOG_INDEX;
+        @SuppressWarnings("unused")
+        long                      reserved;
+        private static final byte MAGIC         = 0x20;
+
+        public SegmentHeader() {
+            super();
+        }
+
+        ByteBuffer encode() {
+            ByteBuffer buffer = ByteBuffer.allocate(HEADER_SIZE);
+            buffer.put(MAGIC);
+            buffer.put(MAGIC);
+            buffer.putLong(this.firstLogIndex);
+            buffer.putLong(RESERVED_FLAG);
+            buffer.flip();
+            return buffer;
+        }
+
+        boolean decode(final ByteBuffer buffer) {
+            if (buffer == null || buffer.remaining() < HEADER_SIZE) {
+                LOG.error("Fail to decode segment header, invalid buffer length: {}",
+                    buffer == null ? 0 : buffer.remaining());
+                return false;
+            }
+            if (buffer.get() != MAGIC) {
+                LOG.error("Fail to decode segment header, invalid magic.");
+                return false;
+            }
+            if (buffer.get() != MAGIC) {
+                LOG.error("Fail to decode segment header, invalid magic.");
+                return false;
+            }
+            this.firstLogIndex = buffer.getLong();
+            return true;
+        }
+    }
+
+    /**
+     * Segment file options.
+     *
+     * @author boyan(boyan@antfin.com)
+     */
+    public static class SegmentFileOptions {
+        // Whether to recover
+        final boolean recover;
+        // Recover start position
+        final int     pos;
+        // True when is the last file.
+        final boolean isLastFile;
+        // True when is a new created file.
+        final boolean isNewFile;
+        final boolean sync;
+
+        private SegmentFileOptions(final boolean recover, final boolean isLastFile, final boolean isNewFile,
+                                   final boolean sync, final int pos) {
+            super();
+            this.isNewFile = isNewFile;
+            this.isLastFile = isLastFile;
+            this.recover = recover;
+            this.sync = sync;
+            this.pos = pos;
+        }
+
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        public static class Builder {
+            boolean recover    = false;
+            int     pos        = 0;
+            boolean isLastFile = false;
+            boolean isNewFile  = false;
+            boolean sync       = true;
+
+            public Builder setRecover(final boolean recover) {
+                this.recover = recover;
+                return this;
+            }
+
+            public Builder setPos(final int pos) {
+                this.pos = pos;
+                return this;
+            }
+
+            public Builder setLastFile(final boolean isLastFile) {
+                this.isLastFile = isLastFile;
+                return this;
+            }
+
+            public Builder setNewFile(final boolean isNewFile) {
+                this.isNewFile = isNewFile;
+                return this;
+            }
+
+            public Builder setSync(final boolean sync) {
+                this.sync = sync;
+                return this;
+            }
+
+            public SegmentFileOptions build() {
+                return new SegmentFileOptions(this.recover, this.isLastFile, this.isNewFile, this.sync, this.pos);
+            }
+        }
+
+    }
+
+    private static final int         BLANK_HOLE_SIZE         = 64;
+
+    private static final Logger      LOG                     = LoggerFactory.getLogger(SegmentFile.class);
+
+    // 4 Bytes for written data length
+    private static final int         RECORD_DATA_LENGTH_SIZE = 4;
+
+    /**
+     * Magic bytes for data buffer.
+     */
+    public static final byte[]       RECORD_MAGIC_BYTES      = new byte[] { (byte) 0x57, (byte) 0x8A };
+
+    public static final int          RECORD_MAGIC_BYTES_SIZE = RECORD_MAGIC_BYTES.length;
+
+    private final SegmentHeader      header;
+
+    // The file last log index(inclusive)
+    private volatile long            lastLogIndex            = Long.MAX_VALUE;
+    // File size
+    private int                      size;
+    // File path
+    private final String             path;
+    // mmap byte buffer.
+    private MappedByteBuffer         buffer;
+    // Wrote position.
+    private volatile int             wrotePos;
+    // Committed position
+    private volatile int             committedPos;
+
+    private final ReadWriteLock      readWriteLock           = new ReentrantReadWriteLock(false);
+
+    private final Lock               writeLock               = this.readWriteLock.writeLock();
+    private final Lock               readLock                = this.readWriteLock.readLock();
+    private final ThreadPoolExecutor writeExecutor;
+    private volatile boolean         swappedOut;
+    private volatile boolean         readOnly;
+    private long                     swappedOutTimestamp     = -1L;
+    private final String             filename;
+
+    public SegmentFile(final int size, final String path, final ThreadPoolExecutor writeExecutor) {
+        super();
+        this.header = new SegmentHeader();
+        this.size = size;
+        this.writeExecutor = writeExecutor;
+        this.path = path;
+        this.filename = FilenameUtils.getName(this.path);
+        this.swappedOut = this.readOnly = false;
+    }
+
+    void setReadOnly(final boolean readOnly) {
+        this.readOnly = readOnly;
+    }
+
+    void setFirstLogIndex(final long index) {
+        this.header.firstLogIndex = index;
+    }
+
+    long getLastLogIndex() {
+        return this.lastLogIndex;
+    }
+
+    @OnlyForTest
+    public int getWrotePos() {
+        return this.wrotePos;
+    }
+
+    int getCommittedPos() {
+        return this.committedPos;
+    }
+
+    String getFilename() {
+        return this.filename;
+    }
+
+    long getFirstLogIndex() {
+        return this.header.firstLogIndex;
+    }
+
+    public boolean isSwappedOut() {
+        return this.swappedOut;
+    }
+
+    int getSize() {
+        return this.size;
+    }
+
+    /**
+     * return true when this segment file is blank that we don't write any data into it.
+     * @return
+     */
+    boolean isBlank() {
+        return this.header.firstLogIndex == BLANK_LOG_INDEX;
+    }
+
+    boolean isHeaderCorrupted() {
+        return this.header == null;
+    }
+
+    String getPath() {
+        return this.path;
+    }
+
+    public void setLastLogIndex(final long lastLogIndex) {
+        this.writeLock.lock();
+        try {
+            this.lastLogIndex = lastLogIndex;
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    private void swapIn() {
+        if (this.swappedOut) {
+            this.writeLock.lock();
+            try {
+                if (!this.swappedOut) {
+                    return;
+                }
+                long startMs = Utils.monotonicMs();
+                mmapFile(false);
+                this.swappedOut = false;
+                LOG.info("Swapped in segment file {} cost {} ms.", this.path, Utils.monotonicMs() - startMs);
+            } finally {
+                this.writeLock.unlock();
+            }
+        }
+    }
+
+    // Cached method for sun.nio.ch.DirectBuffer#address
+    private static MethodHandle ADDRESS_METHOD = null;
+
+    static {
+        try {
+            Class clazz = Class.forName("sun.nio.ch.DirectBuffer");
+            if (clazz != null) {
+                Method method = clazz.getMethod("address");
+                if (method != null) {
+                    ADDRESS_METHOD = MethodHandles.lookup().unreflect(method);
+                }
+            }
+        } catch (Throwable t) {
+            // NOPMD
+        }
+    }
+
+    private Pointer getPointer() {
+        if (ADDRESS_METHOD != null) {
+            try {
+                final long address = (long) ADDRESS_METHOD.invoke(this.buffer);
+                Pointer pointer = new Pointer(address);
+                return pointer;
+            } catch (Throwable t) {
+                // NOPMD
+            }
+        }
+        return null;
+    }
+
+    public void hintLoad() {
+        Pointer pointer = getPointer();
+
+        if (pointer != null) {
+            long beginTime = Utils.monotonicMs();
+            int ret = LibC.INSTANCE.madvise(pointer, new NativeLong(this.size), LibC.MADV_WILLNEED);
+            LOG.info("madvise(MADV_WILLNEED) {} {} {} ret = {} time consuming = {}", pointer, this.path, this.size,
+                ret, Utils.monotonicMs() - beginTime);
+        }
+    }
+
+    public void hintUnload() {
+        Pointer pointer = getPointer();
+
+        if (pointer != null) {
+            long beginTime = Utils.monotonicMs();
+            int ret = LibC.INSTANCE.madvise(pointer, new NativeLong(this.size), LibC.MADV_DONTNEED);
+            LOG.info("madvise(MADV_DONTNEED) {} {} {} ret = {} time consuming = {}", pointer, this.path, this.size,
+                ret, Utils.monotonicMs() - beginTime);
+        }
+    }
+
+    public void swapOut() {
+        if (!this.swappedOut) {
+            this.writeLock.lock();
+            try {
+                if (this.swappedOut) {
+                    return;
+                }
+                if (!this.readOnly) {
+                    LOG.warn("The segment file {} is not readonly, can't be swapped out.", this.path);
+                    return;
+                }
+                final long now = Utils.monotonicMs();
+                if (this.swappedOutTimestamp > 0 && now - this.swappedOutTimestamp < ONE_MINUTE) {
+                    return;
+                }
+                this.swappedOut = true;
+                Utils.unmap(this.buffer);
+                this.buffer = null;
+                this.swappedOutTimestamp = now;
+                LOG.info("Swapped out segment file {} cost {} ms.", this.path, Utils.monotonicMs() - now);
+            } finally {
+                this.writeLock.unlock();
+            }
+        }
+    }
+
+    /**
+     * Truncate data from wrotePos(inclusive) to the file end and set lastLogIndex=logIndex.
+     * @param wrotePos the wrote position(inclusive)
+     * @param logIndex the log index
+     * @param sync whether to call fsync
+     */
+    public void truncateSuffix(final int wrotePos, final long logIndex, final boolean sync) {
+        this.writeLock.lock();
+        try {
+            if (wrotePos >= this.wrotePos) {
+                return;
+            }
+            swapInIfNeed();
+            final int oldPos = this.wrotePos;
+            clear(wrotePos, sync);
+            this.wrotePos = wrotePos;
+            this.lastLogIndex = logIndex;
+            this.buffer.position(wrotePos);
+            LOG.info(
+                "Segment file {} truncate suffix from pos={}, then set lastLogIndex={}, oldWrotePos={}, newWrotePos={}",
+                this.path, wrotePos, logIndex, oldPos, this.wrotePos);
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    /**
+     * Returns true when the segment file contains the log index.
+     *
+     * @param logIndex the log index
+     * @return true if the segment file contains the log index, otherwise return false
+     */
+    public boolean contains(final long logIndex) {
+        this.readLock.lock();
+        try {
+            return logIndex >= this.header.firstLogIndex && logIndex <= this.lastLogIndex;
+        } finally {
+            this.readLock.unlock();
+        }
+    }
+
+    /**
+     * Clear data in [startPos, startPos+64).
+     *
+     * @param startPos the start position(inclusive)
+     */
+    public void clear(final int startPos, final boolean sync) {
+        this.writeLock.lock();
+        try {
+            if (startPos < 0 || startPos > this.size) {
+                return;
+            }
+            final int endPos = Math.min(this.size, startPos + BLANK_HOLE_SIZE);
+            for (int i = startPos; i < endPos; i++) {
+                this.buffer.put(i, (byte) 0);
+            }
+            if (sync) {
+                fsync(this.buffer);
+            }
+            LOG.info("Segment file {} cleared data in [{}, {}).", this.path, startPos, endPos);
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    @Override
+    public boolean init(final SegmentFileOptions opts) {
+        if (opts.isNewFile) {
+            return loadNewFile(opts);
+        } else {
+            return loadExistsFile(opts);
+        }
+
+    }
+
+    private boolean loadNewFile(final SegmentFileOptions opts) {
+        assert (opts.pos == 0);
+        assert (!opts.recover);
+
+        final File file = new File(this.path);
+
+        if (file.exists()) {
+            LOG.error("File {} already exists.", this.path);
+            return false;
+        }
+        long startMs = Utils.monotonicMs();
+        this.writeLock.lock();
+        try (FileChannel fc = openFileChannel(true)) {
+            this.buffer = fc.map(MapMode.READ_WRITE, 0, this.size);
+            // Warmup mmap file
+            this.buffer.position(0);
+            this.buffer.limit(this.size);
+            saveHeader(true);
+
+            this.committedPos = this.wrotePos = HEADER_SIZE;
+            this.buffer.position(this.wrotePos);
+
+            assert (this.wrotePos == this.buffer.position());
+
+            LOG.info("Created a new segment file {} cost {} ms, wrotePosition={}, bufferPosition={}, mappedSize={}.",
+                this.path, Utils.monotonicMs() - startMs, this.wrotePos, this.buffer.position(), this.size);
+            return true;
+        } catch (final IOException e) {
+            LOG.error("Fail to init segment file {}.", this.path, e);
+            return false;
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    private boolean loadExistsFile(final SegmentFileOptions opts) {
+        this.writeLock.lock();
+        try {
+            if (!mmapFile(false)) {
+                return false;
+            }
+            if (!tryRecoverExistsFile(opts)) {
+                return false;
+            }
+            this.readOnly = !opts.isLastFile;
+            return true;
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    private boolean tryRecoverExistsFile(final SegmentFileOptions opts) {
+        try {
+            if (isBlank()) {
+                // A blank segment, we don't need to recover.
+                assert (!opts.recover);
+                this.committedPos = this.wrotePos = HEADER_SIZE;
+                this.buffer.position(this.wrotePos);
+                LOG.info("Segment file {} is blank, truncate it from {}.", this.path, HEADER_SIZE);
+                clear(this.wrotePos, opts.sync);
+            } else {
+                if (opts.recover) {
+                    if (!recover(opts)) {
+                        return false;
+                    }
+                } else {
+                    this.wrotePos = opts.pos;
+                    this.buffer.position(this.wrotePos);
+                }
+                assert (this.wrotePos == this.buffer.position());
+                this.committedPos = this.wrotePos;
+            }
+            LOG.info("Loaded segment file {}, wrotePosition={}, bufferPosition={}, mappedSize={}.", this.path,
+                this.wrotePos, this.buffer.position(), this.size);
+        } catch (final Exception e) {
+            LOG.error("Fail to load segment file {}.", this.path, e);
+            return false;
+        }
+        return true;
+    }
+
+    boolean mmapFile(final boolean create) {
+        if (this.buffer != null) {
+            return true;
+        }
+        final File file = new File(this.path);
+
+        if (file.exists()) {
+            this.size = (int) file.length();
+        } else {
+            if (!create) {
+                LOG.error("File {} is not exists.", this.path);
+                return false;
+            }
+        }
+        try (FileChannel fc = openFileChannel(create)) {
+            this.buffer = fc.map(MapMode.READ_WRITE, 0, this.size);
+            this.buffer.limit(this.size);
+            if (!loadHeader()) {
+                LOG.error("Fail to load segment header from file {}.", this.path);
+                return false;
+            }
+            return true;
+        } catch (final IOException e) {
+            LOG.error("Fail to mmap segment file {}.", this.path, e);
+            return false;
+        }
+    }
+
+    private FileChannel openFileChannel(final boolean create) throws IOException {
+        if (create) {
+            return FileChannel.open(Paths.get(this.path), StandardOpenOption.CREATE, StandardOpenOption.READ,
+                StandardOpenOption.WRITE);
+        } else {
+            return FileChannel.open(Paths.get(this.path), StandardOpenOption.READ, StandardOpenOption.WRITE);
+        }
+    }
+
+    boolean loadHeader() {
+        int oldPos = this.buffer.position();
+        try {
+            this.buffer.position(0);
+            return this.header.decode(this.buffer.asReadOnlyBuffer());
+        } finally {
+            this.buffer.position(oldPos);
+        }
+    }
+
+    void saveHeader(final boolean sync) {
+        int oldPos = this.buffer.position();
+        try {
+            this.buffer.position(0);
+            final ByteBuffer headerBuf = this.header.encode();
+            assert (headerBuf.remaining() == HEADER_SIZE);
+            this.buffer.put(headerBuf);
+            if (sync) {
+                fsync(this.buffer);
+            }
+        } finally {
+            this.buffer.position(oldPos);
+        }
+    }
+
+    @SuppressWarnings("NonAtomicOperationOnVolatileField")
+    private boolean recover(final SegmentFileOptions opts) throws IOException {
+        LOG.info("Start to recover segment file {} from position {}.", this.path, opts.pos);
+        this.wrotePos = opts.pos;
+        if (this.wrotePos < HEADER_SIZE) {
+            this.wrotePos = HEADER_SIZE;
+        }
+        this.buffer.position(this.wrotePos);
+        final long start = Utils.monotonicMs();
+        while (this.wrotePos < this.size) {
+            if (this.buffer.remaining() < RECORD_MAGIC_BYTES_SIZE) {
+                LOG.error("Fail to recover segment file {}, missing magic bytes.", this.path);
+                return false;
+            }
+            final byte[] magicBytes = new byte[RECORD_MAGIC_BYTES_SIZE];
+            this.buffer.get(magicBytes);
+
+            if (!Arrays.equals(RECORD_MAGIC_BYTES, magicBytes)) {
+
+                boolean truncateDirty = false;
+
+                int i = 0;
+                for (final byte b : magicBytes) {
+                    i++;
+                    if (b != 0) {
+                        if (opts.isLastFile) {
+                            // It's the last file
+                            // Truncate the dirty data from wrotePos
+                            LOG.error("Corrupted magic bytes in segment file {} at pos={}, will truncate it.",
+                                this.path, this.wrotePos + i);
+                            truncateDirty = true;
+                            break;
+                        } else {
+                            // It's not the last file, but has invalid magic bytes, the data is corrupted.
+                            LOG.error("Fail to recover segment file {}, invalid magic bytes: {} at pos={}.", this.path,
+                                BytesUtil.toHex(magicBytes), this.wrotePos);
+                            return false;
+                        }
+                    }
+                }
+
+                if (truncateDirty) {
+                    truncateFile(opts.sync);
+                } else {
+                    // Reach blank hole, rewind position.
+                    this.buffer.position(this.buffer.position() - RECORD_MAGIC_BYTES_SIZE);
+                }
+                // Reach end or dirty magic bytes, we should break out.
+                break;
+            }
+
+            if (this.buffer.remaining() < RECORD_DATA_LENGTH_SIZE) {
+                if (opts.isLastFile) {
+                    LOG.error("Corrupted data length in segment file {} at pos={}, will truncate it.", this.path,
+                        this.buffer.position());
+                    truncateFile(opts.sync);
+                    break;
+                } else {
+                    LOG.error(
+                        "Fail to recover segment file {}, invalid data length remaining: {}, expected {} at pos={}.",
+                        this.path, this.buffer.remaining(), RECORD_DATA_LENGTH_SIZE, this.wrotePos);
+                    return false;
+                }
+            }
+
+            final int dataLen = this.buffer.getInt();
+            if (this.buffer.remaining() < dataLen) {
+                if (opts.isLastFile) {
+                    LOG.error(
+                        "Corrupted data in segment file {} at pos={},  expectDataLength={}, but remaining is {}, will truncate it.",
+                        this.path, this.buffer.position(), dataLen, this.buffer.remaining());
+                    truncateFile(opts.sync);
+                    break;
+                } else {
+                    LOG.error(
+                        "Fail to recover segment file {}, invalid data: expected {} bytes in buf but actual {} at pos={}.",
+                        this.path, dataLen, this.buffer.remaining(), this.wrotePos);
+                    return false;
+                }
+
+            }
+            // Skip data
+            this.buffer.position(this.buffer.position() + dataLen);
+            this.wrotePos += RECORD_MAGIC_BYTES_SIZE + RECORD_DATA_LENGTH_SIZE + dataLen;
+        }
+        LOG.info("Recover segment file {} cost {} millis.", this.path, Utils.monotonicMs() - start);
+        return true;
+    }
+
+    private void truncateFile(final boolean sync) throws IOException {
+        // Truncate dirty data.
+        clear(this.wrotePos, sync);
+        this.buffer.position(this.wrotePos);
+        LOG.warn("Truncated segment file {} from pos={}.", this.path, this.wrotePos);
+    }
+
+    public boolean reachesFileEndBy(final long waitToWroteBytes) {
+        this.readLock.lock();
+        try {
+            return this.wrotePos + waitToWroteBytes > this.size;
+        } finally {
+            this.readLock.unlock();
+        }
+    }
+
+    public boolean isFull() {
+        return reachesFileEndBy(1);
+    }
+
+    static int getWriteBytes(final byte[] data) {
+        return RECORD_MAGIC_BYTES_SIZE + RECORD_DATA_LENGTH_SIZE + data.length;
+    }
+
+    /**
+     * Write the data and return it's wrote position.
+     *
+     * @param logIndex the log index
+     * @param data     data to write
+     * @return the wrote position
+     */
+    @SuppressWarnings("NonAtomicOperationOnVolatileField")
+    public int write(final long logIndex, final byte[] data, final WriteContext ctx) {
+        int pos = -1;
+        MappedByteBuffer buf = null;
+        this.writeLock.lock();
+        try {
+            assert (this.wrotePos == this.buffer.position());
+            buf = this.buffer;
+            pos = this.wrotePos;
+            this.wrotePos += RECORD_MAGIC_BYTES_SIZE + RECORD_DATA_LENGTH_SIZE + data.length;
+            this.buffer.position(this.wrotePos);
+            // Update log index.
+            if (isBlank() || pos == HEADER_SIZE) {
+                this.header.firstLogIndex = logIndex;
+                // we don't need to call fsync header here, the new header will be flushed with this wrote.
+                saveHeader(false);
+            }
+            this.lastLogIndex = logIndex;
+            return pos;
+        } finally {
+            this.writeLock.unlock();
+            final int wroteIndex = pos;
+            final MappedByteBuffer buffer = buf;
+            this.writeExecutor.execute(() -> {
+                try {
+                    put(buffer, wroteIndex, RECORD_MAGIC_BYTES);
+                    putInt(buffer, wroteIndex + RECORD_MAGIC_BYTES_SIZE, data.length);
+                    put(buffer, wroteIndex + RECORD_MAGIC_BYTES_SIZE + RECORD_DATA_LENGTH_SIZE, data);
+                } catch (final Exception e) {
+                    ctx.setError(e);
+                } finally {
+                    ctx.finishJob();
+                }
+            });
+        }
+    }
+
+    private static void putInt(final MappedByteBuffer buffer, final int index, final int n) {
+        byte[] bs = new byte[RECORD_DATA_LENGTH_SIZE];
+        Bits.putInt(bs, 0, n);
+        for (int i = 0; i < bs.length; i++) {
+            buffer.put(index + i, bs[i]);
+        }
+    }
+
+    private static void put(final MappedByteBuffer buffer, final int index, final byte[] data) {
+        for (int i = 0; i < data.length; i++) {
+            buffer.put(index + i, data[i]);
+        }
+    }
+
+    /**
+     * Read data from the position.
+     *
+     * @param logIndex the log index
+     * @param pos      the position to read
+     * @return read data
+     */
+    public byte[] read(final long logIndex, final int pos) throws IOException {
+        assert (pos >= HEADER_SIZE);
+        swapInIfNeed();
+        this.readLock.lock();
+        try {
+            if (logIndex < this.header.firstLogIndex || logIndex > this.lastLogIndex) {
+                LOG.warn(
+                    "Try to read data from segment file {} out of range, logIndex={}, readPos={}, firstLogIndex={}, lastLogIndex={}.",
+                    this.path, logIndex, pos, this.header.firstLogIndex, this.lastLogIndex);
+                return null;
+            }
+            if (pos >= this.committedPos) {
+                LOG.warn(
+                    "Try to read data from segment file {} out of comitted position, logIndex={}, readPos={}, wrotePos={}, this.committedPos={}.",
+                    this.path, logIndex, pos, this.wrotePos, this.committedPos);
+                return null;
+            }
+            final ByteBuffer readBuffer = this.buffer.asReadOnlyBuffer();
+            readBuffer.position(pos);
+            if (readBuffer.remaining() < RECORD_MAGIC_BYTES_SIZE) {
+                throw new IOException("Missing magic buffer.");
+            }
+            readBuffer.position(pos + RECORD_MAGIC_BYTES_SIZE);
+            final int dataLen = readBuffer.getInt();
+            //TODO(boyan) reuse data array?
+            final byte[] data = new byte[dataLen];
+            readBuffer.get(data);
+            return data;
+        } finally {
+            this.readLock.unlock();
+        }
+    }
+
+    private void swapInIfNeed() {
+        if (this.swappedOut) {
+            swapIn();
+        }
+    }
+
+    /**
+     * Forces any changes made to this segment file's content to be written to the
+     * storage device containing the mapped file.
+     */
+    public void sync(final boolean sync) throws IOException {
+        MappedByteBuffer buf = null;
+        this.writeLock.lock();
+        try {
+            if (this.committedPos >= this.wrotePos) {
+                return;
+            }
+            this.committedPos = this.wrotePos;
+            buf = this.buffer;
+            LOG.debug("Commit segment file {} at pos {}.", this.path, this.committedPos);
+        } finally {
+            this.writeLock.unlock();
+        }
+        if (sync) {
+            fsync(buf);
+        }
+    }
+
+    private void fsync(final MappedByteBuffer buffer) {
+        if (buffer != null) {
+            long startMs = Utils.monotonicMs();
+            buffer.force();
+            final long cost = Utils.monotonicMs() - startMs;
+            if (cost >= FSYNC_COST_MS_THRESHOLD) {
+                LOG.warn("Call fsync on file {}  cost {} ms.", this.path, cost);
+            }
+        }
+    }
+
+    /**
+     * Destroy the file.
+     */
+    public void destroy() {
+        this.writeLock.lock();
+        try {
+            shutdown();
+            FileUtils.deleteQuietly(new File(this.path));
+            LOG.info("Deleted segment file {}.", this.path);
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        this.writeLock.lock();
+        try {
+            if (this.buffer == null) {
+                return;
+            }
+            hintUnload();
+            Utils.unmap(this.buffer);
+            this.buffer = null;
+            LOG.info("Unloaded segment file {}, current status: {}.", this.path, toString());
+        } finally {
+            this.writeLock.unlock();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "SegmentFile [firstLogIndex=" + this.header.firstLogIndex + ", lastLogIndex=" + this.lastLogIndex
+               + ", size=" + this.size + ", path=" + this.path + ", wrotePos=" + this.wrotePos + ", committedPos="
+               + this.committedPos + "]";
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/Snapshot.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/Snapshot.java
new file mode 100644
index 0000000..2b42d41
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/Snapshot.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.util.Set;
+
+import com.alipay.sofa.jraft.Status;
+import com.google.protobuf.Message;
+
+/**
+ * Represents a state machine snapshot.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-07 10:17:52 AM
+ */
+public abstract class Snapshot extends Status {
+
+    /**
+     * Snapshot metadata file name.
+     */
+    public static final String JRAFT_SNAPSHOT_META_FILE   = "__raft_snapshot_meta";
+    /**
+     * Snapshot file prefix.
+     */
+    public static final String JRAFT_SNAPSHOT_PREFIX      = "snapshot_";
+    /** Snapshot uri scheme for remote peer */
+    public static final String REMOTE_SNAPSHOT_URI_SCHEME = "remote://";
+
+    /**
+     * Get the path of the Snapshot
+     */
+    public abstract String getPath();
+
+    /**
+     * List all the existing files in the Snapshot currently
+     */
+    public abstract Set listFiles();
+
+    /**
+     * Get file meta by fileName.
+     */
+    public abstract Message getFileMeta(final String fileName);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotCopier.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotCopier.java
new file mode 100644
index 0000000..e424e34
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotCopier.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.io.Closeable;
+
+import com.alipay.sofa.jraft.Status;
+
+/**
+ * Copy snapshot from the give resources.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-12 4:55:26 PM
+ */
+public abstract class SnapshotCopier extends Status implements Closeable {
+
+    /**
+     * Cancel the copy job.
+     */
+    public abstract void cancel();
+
+    /**
+     * Block the thread until this copy job finishes, or some error occurs.
+     * @throws InterruptedException if the current thread is interrupted
+     *         while waiting
+     */
+    public abstract void join() throws InterruptedException;
+
+    /**
+     * Start the copy job.
+     */
+    public abstract void start();
+
+    /**
+     * Get the the SnapshotReader which represents the copied Snapshot
+     */
+    public abstract SnapshotReader getReader();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotExecutorImpl.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotExecutorImpl.java
new file mode 100644
index 0000000..13e6200
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotExecutorImpl.java
@@ -0,0 +1,746 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.LoadSnapshotClosure;
+import com.alipay.sofa.jraft.closure.SaveSnapshotClosure;
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.entity.EnumOutter.ErrorType;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.option.SnapshotExecutorOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotResponse;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.SnapshotExecutor;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotStorage;
+import com.alipay.sofa.jraft.util.CountDownEvent;
+import com.alipay.sofa.jraft.util.OnlyForTest;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Snapshot executor implementation.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-22 5:38:56 PM
+ */
+public class SnapshotExecutorImpl implements SnapshotExecutor {
+
+    private static final Logger                        LOG                 = LoggerFactory
+                                                                               .getLogger(SnapshotExecutorImpl.class);
+
+    private final Lock                                 lock                = new ReentrantLock();
+
+    private long                                       lastSnapshotTerm;
+    private long                                       lastSnapshotIndex;
+    private long                                       term;
+    private volatile boolean                           savingSnapshot;
+    private volatile boolean                           loadingSnapshot;
+    private volatile boolean                           stopped;
+    private SnapshotStorage                            snapshotStorage;
+    private SnapshotCopier                             curCopier;
+    private FSMCaller                                  fsmCaller;
+    private NodeImpl                                   node;
+    private LogManager                                 logManager;
+    private final AtomicReference downloadingSnapshot = new AtomicReference<>(null);
+    private SnapshotMeta                               loadingSnapshotMeta;
+    private final CountDownEvent                       runningJobs         = new CountDownEvent();
+
+    /**
+     * Downloading snapshot job.
+     *
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Apr-08 3:07:19 PM
+     */
+    static class DownloadingSnapshot {
+        InstallSnapshotRequest          request;
+        InstallSnapshotResponse.Builder responseBuilder;
+        RpcRequestClosure               done;
+
+        public DownloadingSnapshot(final InstallSnapshotRequest request,
+                                   final InstallSnapshotResponse.Builder responseBuilder, final RpcRequestClosure done) {
+            super();
+            this.request = request;
+            this.responseBuilder = responseBuilder;
+            this.done = done;
+        }
+    }
+
+    /**
+     * Only for test
+     */
+    @OnlyForTest
+    public long getLastSnapshotTerm() {
+        return this.lastSnapshotTerm;
+    }
+
+    /**
+     * Only for test
+     */
+    @OnlyForTest
+    public long getLastSnapshotIndex() {
+        return this.lastSnapshotIndex;
+    }
+
+    /**
+     * Save snapshot done closure
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Apr-08 3:07:52 PM
+     */
+    private class SaveSnapshotDone implements SaveSnapshotClosure {
+
+        SnapshotWriter writer;
+        Closure        done;
+        SnapshotMeta   meta;
+
+        public SaveSnapshotDone(final SnapshotWriter writer, final Closure done, final SnapshotMeta meta) {
+            super();
+            this.writer = writer;
+            this.done = done;
+            this.meta = meta;
+        }
+
+        @Override
+        public void run(final Status status) {
+            Utils.runInThread(() -> continueRun(status));
+        }
+
+        void continueRun(final Status st) {
+            final int ret = onSnapshotSaveDone(st, this.meta, this.writer);
+            if (ret != 0 && st.isOk()) {
+                st.setError(ret, "node call onSnapshotSaveDone failed");
+            }
+            if (this.done != null) {
+                Utils.runClosureInThread(this.done, st);
+            }
+        }
+
+        @Override
+        public SnapshotWriter start(final SnapshotMeta meta) {
+            this.meta = meta;
+            return this.writer;
+        }
+    }
+
+    /**
+     * Install snapshot done closure
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Apr-08 3:08:09 PM
+     */
+    private class InstallSnapshotDone implements LoadSnapshotClosure {
+
+        SnapshotReader reader;
+
+        public InstallSnapshotDone(final SnapshotReader reader) {
+            super();
+            this.reader = reader;
+        }
+
+        @Override
+        public void run(final Status status) {
+            onSnapshotLoadDone(status);
+        }
+
+        @Override
+        public SnapshotReader start() {
+            return this.reader;
+        }
+    }
+
+    /**
+     * Load snapshot at first time closure
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Apr-16 2:57:46 PM
+     */
+    private class FirstSnapshotLoadDone implements LoadSnapshotClosure {
+
+        SnapshotReader reader;
+        CountDownLatch eventLatch;
+        Status         status;
+
+        public FirstSnapshotLoadDone(final SnapshotReader reader) {
+            super();
+            this.reader = reader;
+            this.eventLatch = new CountDownLatch(1);
+        }
+
+        @Override
+        public void run(final Status status) {
+            this.status = status;
+            onSnapshotLoadDone(this.status);
+            this.eventLatch.countDown();
+        }
+
+        public void waitForRun() throws InterruptedException {
+            this.eventLatch.await();
+        }
+
+        @Override
+        public SnapshotReader start() {
+            return this.reader;
+        }
+
+    }
+
+    @Override
+    public boolean init(final SnapshotExecutorOptions opts) {
+        if (StringUtils.isBlank(opts.getUri())) {
+            LOG.error("Snapshot uri is empty.");
+            return false;
+        }
+        this.logManager = opts.getLogManager();
+        this.fsmCaller = opts.getFsmCaller();
+        this.node = opts.getNode();
+        this.term = opts.getInitTerm();
+        this.snapshotStorage = this.node.getServiceFactory().createSnapshotStorage(opts.getUri(),
+            this.node.getRaftOptions());
+        if (opts.isFilterBeforeCopyRemote()) {
+            this.snapshotStorage.setFilterBeforeCopyRemote();
+        }
+        if (opts.getSnapshotThrottle() != null) {
+            this.snapshotStorage.setSnapshotThrottle(opts.getSnapshotThrottle());
+        }
+        if (!this.snapshotStorage.init(null)) {
+            LOG.error("Fail to init snapshot storage.");
+            return false;
+        }
+
+        if (this.snapshotStorage instanceof LocalSnapshotStorage) {
+            final LocalSnapshotStorage tmp = (LocalSnapshotStorage) this.snapshotStorage;
+            if (!tmp.hasServerAddr()) {
+                tmp.setServerAddr(opts.getAddr());
+            }
+        }
+        final SnapshotReader reader = this.snapshotStorage.open();
+        if (reader == null) {
+            return true;
+        }
+        this.loadingSnapshotMeta = reader.load();
+        if (this.loadingSnapshotMeta == null) {
+            LOG.error("Fail to load meta from {}.", opts.getUri());
+            Utils.closeQuietly(reader);
+            return false;
+        }
+        LOG.info("Loading snapshot, meta={}.", this.loadingSnapshotMeta);
+        this.loadingSnapshot = true;
+        this.runningJobs.incrementAndGet();
+        final FirstSnapshotLoadDone done = new FirstSnapshotLoadDone(reader);
+        Requires.requireTrue(this.fsmCaller.onSnapshotLoad(done));
+        try {
+            done.waitForRun();
+        } catch (final InterruptedException e) {
+            LOG.warn("Wait for FirstSnapshotLoadDone run is interrupted.");
+            Thread.currentThread().interrupt();
+            return false;
+        } finally {
+            Utils.closeQuietly(reader);
+        }
+        if (!done.status.isOk()) {
+            LOG.error("Fail to load snapshot from {}, FirstSnapshotLoadDone status is {}.", opts.getUri(), done.status);
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void shutdown() {
+        long savedTerm;
+        this.lock.lock();
+        try {
+            savedTerm = this.term;
+            this.stopped = true;
+        } finally {
+            this.lock.unlock();
+        }
+        interruptDownloadingSnapshots(savedTerm);
+    }
+
+    @Override
+    public NodeImpl getNode() {
+        return this.node;
+    }
+
+    @Override
+    public void doSnapshot(final Closure done) {
+        boolean doUnlock = true;
+        this.lock.lock();
+        try {
+            if (this.stopped) {
+                Utils.runClosureInThread(done, new Status(RaftError.EPERM, "Is stopped."));
+                return;
+            }
+            if (this.downloadingSnapshot.get() != null) {
+                Utils.runClosureInThread(done, new Status(RaftError.EBUSY, "Is loading another snapshot."));
+                return;
+            }
+
+            if (this.savingSnapshot) {
+                Utils.runClosureInThread(done, new Status(RaftError.EBUSY, "Is saving another snapshot."));
+                return;
+            }
+
+            if (this.fsmCaller.getLastAppliedIndex() == this.lastSnapshotIndex) {
+                // There might be false positive as the getLastAppliedIndex() is being
+                // updated. But it's fine since we will do next snapshot saving in a
+                // predictable time.
+                doUnlock = false;
+                this.lock.unlock();
+                this.logManager.clearBufferedLogs();
+                Utils.runClosureInThread(done);
+                return;
+            }
+
+            final long distance = this.fsmCaller.getLastAppliedIndex() - this.lastSnapshotIndex;
+            if (distance < this.node.getOptions().getSnapshotLogIndexMargin()) {
+                // If state machine's lastAppliedIndex value minus lastSnapshotIndex value is
+                // less than snapshotLogIndexMargin value, then directly return.
+                if (this.node != null) {
+                    LOG.debug(
+                        "Node {} snapshotLogIndexMargin={}, distance={}, so ignore this time of snapshot by snapshotLogIndexMargin setting.",
+                        this.node.getNodeId(), distance, this.node.getOptions().getSnapshotLogIndexMargin());
+                }
+                doUnlock = false;
+                this.lock.unlock();
+                Utils.runClosureInThread(done);
+                return;
+            }
+
+            final SnapshotWriter writer = this.snapshotStorage.create();
+            if (writer == null) {
+                Utils.runClosureInThread(done, new Status(RaftError.EIO, "Fail to create writer."));
+                reportError(RaftError.EIO.getNumber(), "Fail to create snapshot writer.");
+                return;
+            }
+            this.savingSnapshot = true;
+            final SaveSnapshotDone saveSnapshotDone = new SaveSnapshotDone(writer, done, null);
+            if (!this.fsmCaller.onSnapshotSave(saveSnapshotDone)) {
+                Utils.runClosureInThread(done, new Status(RaftError.EHOSTDOWN, "The raft node is down."));
+                return;
+            }
+            this.runningJobs.incrementAndGet();
+        } finally {
+            if (doUnlock) {
+                this.lock.unlock();
+            }
+        }
+
+    }
+
+    int onSnapshotSaveDone(final Status st, final SnapshotMeta meta, final SnapshotWriter writer) {
+        int ret;
+        this.lock.lock();
+        try {
+            ret = st.getCode();
+            // InstallSnapshot can break SaveSnapshot, check InstallSnapshot when SaveSnapshot
+            // because upstream Snapshot maybe newer than local Snapshot.
+            if (st.isOk()) {
+                if (meta.getLastIncludedIndex() <= this.lastSnapshotIndex) {
+                    ret = RaftError.ESTALE.getNumber();
+                    if (this.node != null) {
+                        LOG.warn("Node {} discards an stale snapshot lastIncludedIndex={}, lastSnapshotIndex={}.",
+                            this.node.getNodeId(), meta.getLastIncludedIndex(), this.lastSnapshotIndex);
+                    }
+                    writer.setError(RaftError.ESTALE, "Installing snapshot is older than local snapshot");
+                }
+            }
+        } finally {
+            this.lock.unlock();
+        }
+
+        if (ret == 0) {
+            if (!writer.saveMeta(meta)) {
+                LOG.warn("Fail to save snapshot {}.", writer.getPath());
+                ret = RaftError.EIO.getNumber();
+            }
+        } else {
+            if (writer.isOk()) {
+                writer.setError(ret, "Fail to do snapshot.");
+            }
+        }
+        try {
+            writer.close();
+        } catch (final IOException e) {
+            LOG.error("Fail to close writer", e);
+            ret = RaftError.EIO.getNumber();
+        }
+        boolean doUnlock = true;
+        this.lock.lock();
+        try {
+            if (ret == 0) {
+                this.lastSnapshotIndex = meta.getLastIncludedIndex();
+                this.lastSnapshotTerm = meta.getLastIncludedTerm();
+                doUnlock = false;
+                this.lock.unlock();
+                this.logManager.setSnapshot(meta); // should be out of lock
+                doUnlock = true;
+                this.lock.lock();
+            }
+            if (ret == RaftError.EIO.getNumber()) {
+                reportError(RaftError.EIO.getNumber(), "Fail to save snapshot.");
+            }
+            this.savingSnapshot = false;
+            this.runningJobs.countDown();
+            return ret;
+
+        } finally {
+            if (doUnlock) {
+                this.lock.unlock();
+            }
+        }
+    }
+
+    private void onSnapshotLoadDone(final Status st) {
+        DownloadingSnapshot m;
+        boolean doUnlock = true;
+        this.lock.lock();
+        try {
+            Requires.requireTrue(this.loadingSnapshot, "Not loading snapshot");
+            m = this.downloadingSnapshot.get();
+            if (st.isOk()) {
+                this.lastSnapshotIndex = this.loadingSnapshotMeta.getLastIncludedIndex();
+                this.lastSnapshotTerm = this.loadingSnapshotMeta.getLastIncludedTerm();
+                doUnlock = false;
+                this.lock.unlock();
+                this.logManager.setSnapshot(this.loadingSnapshotMeta); // should be out of lock
+                doUnlock = true;
+                this.lock.lock();
+            }
+            final StringBuilder sb = new StringBuilder();
+            if (this.node != null) {
+                sb.append("Node ").append(this.node.getNodeId()).append(" ");
+            }
+            sb.append("onSnapshotLoadDone, ").append(this.loadingSnapshotMeta);
+            LOG.info(sb.toString());
+            doUnlock = false;
+            this.lock.unlock();
+            if (this.node != null) {
+                this.node.updateConfigurationAfterInstallingSnapshot();
+            }
+            doUnlock = true;
+            this.lock.lock();
+            this.loadingSnapshot = false;
+            this.downloadingSnapshot.set(null);
+
+        } finally {
+            if (doUnlock) {
+                this.lock.unlock();
+            }
+        }
+        if (m != null) {
+            // Respond RPC
+            if (!st.isOk()) {
+                m.done.run(st);
+            } else {
+                m.responseBuilder.setSuccess(true);
+                m.done.sendResponse(m.responseBuilder.build());
+            }
+        }
+        this.runningJobs.countDown();
+    }
+
+    @Override
+    public void installSnapshot(final InstallSnapshotRequest request, final InstallSnapshotResponse.Builder response,
+                                final RpcRequestClosure done) {
+        final SnapshotMeta meta = request.getMeta();
+        final DownloadingSnapshot ds = new DownloadingSnapshot(request, response, done);
+        // DON'T access request, response, and done after this point
+        // as the retry snapshot will replace this one.
+        if (!registerDownloadingSnapshot(ds)) {
+            LOG.warn("Fail to register downloading snapshot.");
+            // This RPC will be responded by the previous session
+            return;
+        }
+        Requires.requireNonNull(this.curCopier, "curCopier");
+        try {
+            this.curCopier.join();
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt();
+            LOG.warn("Install snapshot copy job was canceled.");
+            return;
+        }
+
+        loadDownloadingSnapshot(ds, meta);
+    }
+
+    void loadDownloadingSnapshot(final DownloadingSnapshot ds, final SnapshotMeta meta) {
+        SnapshotReader reader;
+        this.lock.lock();
+        try {
+            if (ds != this.downloadingSnapshot.get()) {
+                // It is interrupted and response by other request,just return
+                return;
+            }
+            Requires.requireNonNull(this.curCopier, "curCopier");
+            reader = this.curCopier.getReader();
+            if (!this.curCopier.isOk()) {
+                if (this.curCopier.getCode() == RaftError.EIO.getNumber()) {
+                    reportError(this.curCopier.getCode(), this.curCopier.getErrorMsg());
+                }
+                Utils.closeQuietly(reader);
+                ds.done.run(this.curCopier);
+                Utils.closeQuietly(this.curCopier);
+                this.curCopier = null;
+                this.downloadingSnapshot.set(null);
+                this.runningJobs.countDown();
+                return;
+            }
+            Utils.closeQuietly(this.curCopier);
+            this.curCopier = null;
+            if (reader == null || !reader.isOk()) {
+                Utils.closeQuietly(reader);
+                this.downloadingSnapshot.set(null);
+                ds.done.sendResponse(RpcFactoryHelper //
+                    .responseFactory() //
+                    .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINTERNAL,
+                        "Fail to copy snapshot from %s", ds.request.getUri()));
+                this.runningJobs.countDown();
+                return;
+            }
+            this.loadingSnapshot = true;
+            this.loadingSnapshotMeta = meta;
+        } finally {
+            this.lock.unlock();
+        }
+        final InstallSnapshotDone installSnapshotDone = new InstallSnapshotDone(reader);
+        if (!this.fsmCaller.onSnapshotLoad(installSnapshotDone)) {
+            LOG.warn("Fail to call fsm onSnapshotLoad.");
+            installSnapshotDone.run(new Status(RaftError.EHOSTDOWN, "This raft node is down"));
+        }
+    }
+
+    @SuppressWarnings("all")
+    boolean registerDownloadingSnapshot(final DownloadingSnapshot ds) {
+        DownloadingSnapshot saved = null;
+        boolean result = true;
+
+        this.lock.lock();
+        try {
+            if (this.stopped) {
+                LOG.warn("Register DownloadingSnapshot failed: node is stopped.");
+                ds.done
+                    .sendResponse(RpcFactoryHelper //
+                        .responseFactory() //
+                        .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EHOSTDOWN,
+                            "Node is stopped."));
+                return false;
+            }
+            if (this.savingSnapshot) {
+                LOG.warn("Register DownloadingSnapshot failed: is saving snapshot.");
+                ds.done.sendResponse(RpcFactoryHelper //
+                    .responseFactory().newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EBUSY,
+                        "Node is saving snapshot."));
+                return false;
+            }
+
+            ds.responseBuilder.setTerm(this.term);
+            if (ds.request.getTerm() != this.term) {
+                LOG.warn("Register DownloadingSnapshot failed: term mismatch, expect {} but {}.", this.term,
+                    ds.request.getTerm());
+                ds.responseBuilder.setSuccess(false);
+                ds.done.sendResponse(ds.responseBuilder.build());
+                return false;
+            }
+            if (ds.request.getMeta().getLastIncludedIndex() <= this.lastSnapshotIndex) {
+                LOG.warn(
+                    "Register DownloadingSnapshot failed: snapshot is not newer, request lastIncludedIndex={}, lastSnapshotIndex={}.",
+                    ds.request.getMeta().getLastIncludedIndex(), this.lastSnapshotIndex);
+                ds.responseBuilder.setSuccess(true);
+                ds.done.sendResponse(ds.responseBuilder.build());
+                return false;
+            }
+            final DownloadingSnapshot m = this.downloadingSnapshot.get();
+            if (m == null) {
+                this.downloadingSnapshot.set(ds);
+                Requires.requireTrue(this.curCopier == null, "Current copier is not null");
+                this.curCopier = this.snapshotStorage.startToCopyFrom(ds.request.getUri(), newCopierOpts());
+                if (this.curCopier == null) {
+                    this.downloadingSnapshot.set(null);
+                    LOG.warn("Register DownloadingSnapshot failed: fail to copy file from {}.", ds.request.getUri());
+                    ds.done.sendResponse(RpcFactoryHelper //
+                        .responseFactory() //
+                        .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINVAL,
+                            "Fail to copy from: %s", ds.request.getUri()));
+                    return false;
+                }
+                this.runningJobs.incrementAndGet();
+                return true;
+            }
+
+            // A previous snapshot is under installing, check if this is the same
+            // snapshot and resume it, otherwise drop previous snapshot as this one is
+            // newer
+
+            if (m.request.getMeta().getLastIncludedIndex() == ds.request.getMeta().getLastIncludedIndex()) {
+                // m is a retry
+                // Copy |*ds| to |*m| so that the former session would respond
+                // this RPC.
+                saved = m;
+                this.downloadingSnapshot.set(ds);
+                result = true;
+            } else if (m.request.getMeta().getLastIncludedIndex() > ds.request.getMeta().getLastIncludedIndex()) {
+                // |is| is older
+                LOG.warn("Register DownloadingSnapshot failed: is installing a newer one, lastIncludeIndex={}.",
+                    m.request.getMeta().getLastIncludedIndex());
+                ds.done.sendResponse(RpcFactoryHelper //
+                    .responseFactory() //
+                    .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINVAL,
+                        "A newer snapshot is under installing"));
+                return false;
+            } else {
+                // |is| is newer
+                if (this.loadingSnapshot) {
+                    LOG.warn("Register DownloadingSnapshot failed: is loading an older snapshot, lastIncludeIndex={}.",
+                        m.request.getMeta().getLastIncludedIndex());
+                    ds.done.sendResponse(RpcFactoryHelper //
+                        .responseFactory() //
+                        .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EBUSY,
+                            "A former snapshot is under loading"));
+                    return false;
+                }
+                Requires.requireNonNull(this.curCopier, "curCopier");
+                this.curCopier.cancel();
+                LOG.warn(
+                    "Register DownloadingSnapshot failed: an older snapshot is under installing, cancel downloading, lastIncludeIndex={}.",
+                    m.request.getMeta().getLastIncludedIndex());
+                ds.done.sendResponse(RpcFactoryHelper //
+                    .responseFactory() //
+                    .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EBUSY,
+                        "A former snapshot is under installing, trying to cancel"));
+                return false;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        if (saved != null) {
+            // Respond replaced session
+            LOG.warn("Register DownloadingSnapshot failed: interrupted by retry installing request.");
+            saved.done.sendResponse(RpcFactoryHelper //
+                .responseFactory() //
+                .newResponse(InstallSnapshotResponse.getDefaultInstance(), RaftError.EINTR,
+                    "Interrupted by the retry InstallSnapshotRequest"));
+        }
+        return result;
+    }
+
+    private SnapshotCopierOptions newCopierOpts() {
+        final SnapshotCopierOptions copierOpts = new SnapshotCopierOptions();
+        copierOpts.setNodeOptions(this.node.getOptions());
+        copierOpts.setRaftClientService(this.node.getRpcService());
+        copierOpts.setTimerManager(this.node.getTimerManager());
+        copierOpts.setRaftOptions(this.node.getRaftOptions());
+        return copierOpts;
+    }
+
+    @Override
+    public void interruptDownloadingSnapshots(final long newTerm) {
+        this.lock.lock();
+        try {
+            Requires.requireTrue(newTerm >= this.term);
+            this.term = newTerm;
+            if (this.downloadingSnapshot.get() == null) {
+                return;
+            }
+            if (this.loadingSnapshot) {
+                // We can't interrupt loading
+                return;
+            }
+            Requires.requireNonNull(this.curCopier, "curCopier");
+            this.curCopier.cancel();
+            LOG.info("Trying to cancel downloading snapshot: {}.", this.downloadingSnapshot.get().request);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    private void reportError(final int errCode, final String fmt, final Object... args) {
+        final RaftException error = new RaftException(ErrorType.ERROR_TYPE_SNAPSHOT);
+        error.setStatus(new Status(errCode, fmt, args));
+        this.fsmCaller.onError(error);
+    }
+
+    @Override
+    public boolean isInstallingSnapshot() {
+        return this.downloadingSnapshot.get() != null;
+    }
+
+    @Override
+    public SnapshotStorage getSnapshotStorage() {
+        return this.snapshotStorage;
+    }
+
+    @Override
+    public void join() throws InterruptedException {
+        this.runningJobs.await();
+    }
+
+    @Override
+    public void describe(final Printer out) {
+        final long _lastSnapshotTerm;
+        final long _lastSnapshotIndex;
+        final long _term;
+        final boolean _savingSnapshot;
+        final boolean _loadingSnapshot;
+        final boolean _stopped;
+        this.lock.lock();
+        try {
+            _lastSnapshotTerm = this.lastSnapshotTerm;
+            _lastSnapshotIndex = this.lastSnapshotIndex;
+            _term = this.term;
+            _savingSnapshot = this.savingSnapshot;
+            _loadingSnapshot = this.loadingSnapshot;
+            _stopped = this.stopped;
+        } finally {
+            this.lock.unlock();
+        }
+        out.print("  lastSnapshotTerm: ") //
+            .println(_lastSnapshotTerm);
+        out.print("  lastSnapshotIndex: ") //
+            .println(_lastSnapshotIndex);
+        out.print("  term: ") //
+            .println(_term);
+        out.print("  savingSnapshot: ") //
+            .println(_savingSnapshot);
+        out.print("  loadingSnapshot: ") //
+            .println(_loadingSnapshot);
+        out.print("  stopped: ") //
+            .println(_stopped);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotReader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotReader.java
new file mode 100644
index 0000000..b04d312
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotReader.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.io.Closeable;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+
+/**
+ * Snapshot reader.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-12 4:53:40 PM
+ */
+public abstract class SnapshotReader extends Snapshot implements Closeable, Lifecycle {
+
+    /**
+     * Load the snapshot metadata.
+     */
+    public abstract SnapshotMeta load();
+
+    /**
+     * Generate uri for other peers to copy this snapshot.
+     * Return an empty string if some error has occur.
+     */
+    public abstract String generateURIForCopy();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotWriter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotWriter.java
new file mode 100644
index 0000000..e031c2c
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/SnapshotWriter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.google.protobuf.Message;
+
+/**
+ * Snapshot writer.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-12 4:52:10 PM
+ */
+public abstract class SnapshotWriter extends Snapshot implements Closeable, Lifecycle {
+
+    /**
+     * Save a snapshot metadata.
+     *
+     * @param meta snapshot metadata
+     * @return true on success
+     */
+    public abstract boolean saveMeta(final SnapshotMeta meta);
+
+    /**
+     * Adds a snapshot file without metadata.
+     *
+     * @param fileName file name
+     * @return true on success
+     */
+    public boolean addFile(final String fileName) {
+        return addFile(fileName, null);
+    }
+
+    /**
+     * Adds a snapshot file with metadata.
+     *
+     * @param fileName file name
+     * @param fileMeta file metadata
+     * @return true on success
+     */
+    public abstract boolean addFile(final String fileName, final Message fileMeta);
+
+    /**
+     * Remove a snapshot file.
+     *
+     * @param fileName file name
+     * @return true on success
+     */
+    public abstract boolean removeFile(final String fileName);
+
+    /**
+     * Close the writer.
+     *
+     * @param keepDataOnError whether to keep data when error happens.
+     * @throws IOException if occurred an IO error
+     */
+    public abstract void close(final boolean keepDataOnError) throws IOException;
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottle.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottle.java
new file mode 100644
index 0000000..5eb2954
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottle.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * SnapshotThrottle with throughput threshold used in installSnapshot.
+ *
+ * @author dennis
+ */
+public class ThroughputSnapshotThrottle implements SnapshotThrottle {
+
+    private final long throttleThroughputBytes;
+    private final long checkCycleSecs;
+    private long       lastThroughputCheckTimeUs;
+    private long       currThroughputBytes;
+    private final Lock lock = new ReentrantLock();
+    private final long baseAligningTimeUs;
+
+    public ThroughputSnapshotThrottle(final long throttleThroughputBytes, final long checkCycleSecs) {
+        this.throttleThroughputBytes = throttleThroughputBytes;
+        this.checkCycleSecs = checkCycleSecs;
+        this.currThroughputBytes = 0L;
+        this.baseAligningTimeUs = 1000 * 1000 / checkCycleSecs;
+        this.lastThroughputCheckTimeUs = this.calculateCheckTimeUs(Utils.monotonicUs());
+    }
+
+    private long calculateCheckTimeUs(final long currTimeUs) {
+        return currTimeUs / this.baseAligningTimeUs * this.baseAligningTimeUs;
+    }
+
+    @Override
+    public long throttledByThroughput(final long bytes) {
+        long availableSize;
+        final long nowUs = Utils.monotonicUs();
+        final long limitPerCycle = this.throttleThroughputBytes / this.checkCycleSecs;
+        this.lock.lock();
+        try {
+            if (this.currThroughputBytes + bytes > limitPerCycle) {
+                // reading another |bytes| exceeds the limit
+                if (nowUs - this.lastThroughputCheckTimeUs <= 1000 * 1000 / this.checkCycleSecs) {
+                    // if time interval is less than or equal to a cycle, read more data
+                    // to make full use of the throughput of current cycle.
+                    availableSize = limitPerCycle - this.currThroughputBytes;
+                    this.currThroughputBytes = limitPerCycle;
+                } else {
+                    // otherwise, read the data in the next cycle.
+                    availableSize = bytes > limitPerCycle ? limitPerCycle : bytes;
+                    this.currThroughputBytes = availableSize;
+                    this.lastThroughputCheckTimeUs = calculateCheckTimeUs(nowUs);
+                }
+            } else {
+                // reading another |bytes| doesn't exceed limit(less than or equal to),
+                // put it in current cycle
+                availableSize = bytes;
+                this.currThroughputBytes += availableSize;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return availableSize;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshot.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshot.java
new file mode 100644
index 0000000..ed5d513
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshot.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.util.Set;
+
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.google.protobuf.Message;
+
+/**
+ * Describe the Snapshot on another machine.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-07 10:21:55 AM
+ */
+public class LocalSnapshot extends Snapshot {
+
+    private final LocalSnapshotMetaTable metaTable;
+
+    public LocalSnapshot(RaftOptions raftOptions) {
+        this.metaTable = new LocalSnapshotMetaTable(raftOptions);
+    }
+
+    public LocalSnapshotMetaTable getMetaTable() {
+        return this.metaTable;
+    }
+
+    @Override
+    public String getPath() {
+        throw new UnsupportedOperationException();
+    }
+
+    /**
+     * List all the existing files in the Snapshot currently
+     *
+     * @return the existing file list
+     */
+    @Override
+    public Set listFiles() {
+        return this.metaTable.listFiles();
+    }
+
+    @Override
+    public Message getFileMeta(final String fileName) {
+        return this.metaTable.getFileMeta(fileName);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopier.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopier.java
new file mode 100644
index 0000000..bd9d81c
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopier.java
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.Future;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.FileSource;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotCopier;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.remote.RemoteFileCopier;
+import com.alipay.sofa.jraft.storage.snapshot.remote.Session;
+import com.alipay.sofa.jraft.util.ArrayDeque;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Copy another machine snapshot to local.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-07 11:32:30 AM
+ */
+public class LocalSnapshotCopier extends SnapshotCopier {
+
+    private static final Logger          LOG  = LoggerFactory.getLogger(LocalSnapshotCopier.class);
+
+    private final Lock                   lock = new ReentrantLock();
+    /** The copy job future object*/
+    private volatile Future           future;
+    private boolean                      cancelled;
+    /** snapshot writer */
+    private LocalSnapshotWriter          writer;
+    /** snapshot reader */
+    private volatile LocalSnapshotReader reader;
+    /** snapshot storage*/
+    private LocalSnapshotStorage         storage;
+    private boolean                      filterBeforeCopyRemote;
+    private LocalSnapshot                remoteSnapshot;
+    /** remote file copier*/
+    private RemoteFileCopier             copier;
+    /** current copying session*/
+    private Session                      curSession;
+    private SnapshotThrottle             snapshotThrottle;
+
+    public void setSnapshotThrottle(final SnapshotThrottle snapshotThrottle) {
+        this.snapshotThrottle = snapshotThrottle;
+    }
+
+    private void startCopy() {
+        try {
+            internalCopy();
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt(); //reset/ignore
+        } catch (final IOException e) {
+            LOG.error("Fail to start copy job", e);
+        }
+    }
+
+    private void internalCopy() throws IOException, InterruptedException {
+        // noinspection ConstantConditions
+        do {
+            loadMetaTable();
+            if (!isOk()) {
+                break;
+            }
+            filter();
+            if (!isOk()) {
+                break;
+            }
+            final Set files = this.remoteSnapshot.listFiles();
+            for (final String file : files) {
+                copyFile(file);
+            }
+        } while (false);
+        if (!isOk() && this.writer != null && this.writer.isOk()) {
+            this.writer.setError(getCode(), getErrorMsg());
+        }
+        if (this.writer != null) {
+            Utils.closeQuietly(this.writer);
+            this.writer = null;
+        }
+        if (isOk()) {
+            this.reader = (LocalSnapshotReader) this.storage.open();
+        }
+    }
+
+    void copyFile(final String fileName) throws IOException, InterruptedException {
+        if (this.writer.getFileMeta(fileName) != null) {
+            LOG.info("Skipped downloading {}", fileName);
+            return;
+        }
+        if (!checkFile(fileName)) {
+            return;
+        }
+        final String filePath = this.writer.getPath() + File.separator + fileName;
+        final Path subPath = Paths.get(filePath);
+        if (!subPath.equals(subPath.getParent()) && !subPath.getParent().getFileName().toString().equals(".")) {
+            final File parentDir = subPath.getParent().toFile();
+            if (!parentDir.exists() && !parentDir.mkdirs()) {
+                LOG.error("Fail to create directory for {}", filePath);
+                setError(RaftError.EIO, "Fail to create directory");
+                return;
+            }
+        }
+
+        final LocalFileMeta meta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
+        Session session = null;
+        try {
+            this.lock.lock();
+            try {
+                if (this.cancelled) {
+                    if (isOk()) {
+                        setError(RaftError.ECANCELED, "ECANCELED");
+                    }
+                    return;
+                }
+                session = this.copier.startCopyToFile(fileName, filePath, null);
+                if (session == null) {
+                    LOG.error("Fail to copy {}", fileName);
+                    setError(-1, "Fail to copy %s", fileName);
+                    return;
+                }
+                this.curSession = session;
+
+            } finally {
+                this.lock.unlock();
+            }
+            session.join(); // join out of lock
+            this.lock.lock();
+            try {
+                this.curSession = null;
+            } finally {
+                this.lock.unlock();
+            }
+            if (!session.status().isOk() && isOk()) {
+                setError(session.status().getCode(), session.status().getErrorMsg());
+                return;
+            }
+            if (!this.writer.addFile(fileName, meta)) {
+                setError(RaftError.EIO, "Fail to add file to writer");
+                return;
+            }
+            if (!this.writer.sync()) {
+                setError(RaftError.EIO, "Fail to sync writer");
+            }
+        } finally {
+            if (session != null) {
+                Utils.closeQuietly(session);
+            }
+        }
+    }
+
+    private boolean checkFile(final String fileName) {
+        try {
+            final String parentCanonicalPath = Paths.get(this.writer.getPath()).toFile().getCanonicalPath();
+            final Path filePath = Paths.get(parentCanonicalPath, fileName);
+            final File file = filePath.toFile();
+            final String fileAbsolutePath = file.getAbsolutePath();
+            final String fileCanonicalPath = file.getCanonicalPath();
+            if (!fileAbsolutePath.equals(fileCanonicalPath)) {
+                LOG.error("File[{}] are not allowed to be created outside of directory[{}].", fileAbsolutePath,
+                    fileCanonicalPath);
+                setError(RaftError.EIO, "File[%s] are not allowed to be created outside of directory.",
+                    fileAbsolutePath, fileCanonicalPath);
+                return false;
+            }
+        } catch (final IOException e) {
+            LOG.error("Failed to check file: {}, writer path: {}.", fileName, this.writer.getPath(), e);
+            setError(RaftError.EIO, "Failed to check file: {}, writer path: {}.", fileName, this.writer.getPath());
+            return false;
+        }
+        return true;
+    }
+
+    private void loadMetaTable() throws InterruptedException {
+        final ByteBufferCollector metaBuf = ByteBufferCollector.allocate(0);
+        Session session = null;
+        try {
+            this.lock.lock();
+            try {
+                if (this.cancelled) {
+                    if (isOk()) {
+                        setError(RaftError.ECANCELED, "ECANCELED");
+                    }
+                    return;
+                }
+                session = this.copier.startCopy2IoBuffer(Snapshot.JRAFT_SNAPSHOT_META_FILE, metaBuf, null);
+                this.curSession = session;
+            } finally {
+                this.lock.unlock();
+            }
+            session.join(); //join out of lock.
+            this.lock.lock();
+            try {
+                this.curSession = null;
+            } finally {
+                this.lock.unlock();
+            }
+            if (!session.status().isOk() && isOk()) {
+                LOG.warn("Fail to copy meta file: {}", session.status());
+                setError(session.status().getCode(), session.status().getErrorMsg());
+                return;
+            }
+            if (!this.remoteSnapshot.getMetaTable().loadFromIoBufferAsRemote(metaBuf.getBuffer())) {
+                LOG.warn("Bad meta_table format");
+                setError(-1, "Bad meta_table format from remote");
+                return;
+            }
+            Requires.requireTrue(this.remoteSnapshot.getMetaTable().hasMeta(), "Invalid remote snapshot meta:%s",
+                this.remoteSnapshot.getMetaTable().getMeta());
+        } finally {
+            if (session != null) {
+                Utils.closeQuietly(session);
+            }
+        }
+    }
+
+    boolean filterBeforeCopy(final LocalSnapshotWriter writer, final SnapshotReader lastSnapshot) throws IOException {
+        final Set existingFiles = writer.listFiles();
+        final ArrayDeque toRemove = new ArrayDeque<>();
+        for (final String file : existingFiles) {
+            if (this.remoteSnapshot.getFileMeta(file) == null) {
+                toRemove.add(file);
+                writer.removeFile(file);
+            }
+        }
+
+        final Set remoteFiles = this.remoteSnapshot.listFiles();
+
+        for (final String fileName : remoteFiles) {
+            final LocalFileMeta remoteMeta = (LocalFileMeta) this.remoteSnapshot.getFileMeta(fileName);
+            Requires.requireNonNull(remoteMeta, "remoteMeta");
+            if (!remoteMeta.hasChecksum()) {
+                // Re-download file if this file doesn't have checksum
+                writer.removeFile(fileName);
+                toRemove.add(fileName);
+                continue;
+            }
+
+            LocalFileMeta localMeta = (LocalFileMeta) writer.getFileMeta(fileName);
+            if (localMeta != null) {
+                if (localMeta.hasChecksum() && localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
+                    LOG.info("Keep file={} checksum={} in {}", fileName, remoteMeta.getChecksum(), writer.getPath());
+                    continue;
+                }
+                // Remove files from writer so that the file is to be copied from
+                // remote_snapshot or last_snapshot
+                writer.removeFile(fileName);
+                toRemove.add(fileName);
+            }
+            // Try find files in last_snapshot
+            if (lastSnapshot == null) {
+                continue;
+            }
+            if ((localMeta = (LocalFileMeta) lastSnapshot.getFileMeta(fileName)) == null) {
+                continue;
+            }
+            if (!localMeta.hasChecksum() || !localMeta.getChecksum().equals(remoteMeta.getChecksum())) {
+                continue;
+            }
+
+            LOG.info("Found the same file ={} checksum={} in lastSnapshot={}", fileName, remoteMeta.getChecksum(),
+                lastSnapshot.getPath());
+            if (localMeta.getSource() == FileSource.FILE_SOURCE_LOCAL) {
+                final String sourcePath = lastSnapshot.getPath() + File.separator + fileName;
+                final String destPath = writer.getPath() + File.separator + fileName;
+                FileUtils.deleteQuietly(new File(destPath));
+                try {
+                    Files.createLink(Paths.get(destPath), Paths.get(sourcePath));
+                } catch (final IOException e) {
+                    LOG.error("Fail to link {} to {}", sourcePath, destPath, e);
+                    continue;
+                }
+                // Don't delete linked file
+                if (!toRemove.isEmpty() && toRemove.peekLast().equals(fileName)) {
+                    toRemove.pollLast();
+                }
+            }
+            // Copy file from last_snapshot
+            writer.addFile(fileName, localMeta);
+        }
+        if (!writer.sync()) {
+            LOG.error("Fail to sync writer on path={}", writer.getPath());
+            return false;
+        }
+        for (final String fileName : toRemove) {
+            final String removePath = writer.getPath() + File.separator + fileName;
+            FileUtils.deleteQuietly(new File(removePath));
+            LOG.info("Deleted file: {}", removePath);
+        }
+        return true;
+    }
+
+    private void filter() throws IOException {
+        this.writer = (LocalSnapshotWriter) this.storage.create(!this.filterBeforeCopyRemote);
+        if (this.writer == null) {
+            setError(RaftError.EIO, "Fail to create snapshot writer");
+            return;
+        }
+        if (this.filterBeforeCopyRemote) {
+            final SnapshotReader reader = this.storage.open();
+            if (!filterBeforeCopy(this.writer, reader)) {
+                LOG.warn("Fail to filter writer before copying, destroy and create a new writer.");
+                this.writer.setError(-1, "Fail to filter");
+                Utils.closeQuietly(this.writer);
+                this.writer = (LocalSnapshotWriter) this.storage.create(true);
+            }
+            if (reader != null) {
+                Utils.closeQuietly(reader);
+            }
+            if (this.writer == null) {
+                setError(RaftError.EIO, "Fail to create snapshot writer");
+                return;
+            }
+        }
+        this.writer.saveMeta(this.remoteSnapshot.getMetaTable().getMeta());
+        if (!this.writer.sync()) {
+            LOG.error("Fail to sync snapshot writer path={}", this.writer.getPath());
+            setError(RaftError.EIO, "Fail to sync snapshot writer");
+        }
+    }
+
+    public boolean init(final String uri, final SnapshotCopierOptions opts) {
+        this.copier = new RemoteFileCopier();
+        this.cancelled = false;
+        this.filterBeforeCopyRemote = opts.getNodeOptions().isFilterBeforeCopyRemote();
+        this.remoteSnapshot = new LocalSnapshot(opts.getRaftOptions());
+        return this.copier.init(uri, this.snapshotThrottle, opts);
+    }
+
+    public SnapshotStorage getStorage() {
+        return this.storage;
+    }
+
+    public void setStorage(final SnapshotStorage storage) {
+        this.storage = (LocalSnapshotStorage) storage;
+    }
+
+    public boolean isFilterBeforeCopyRemote() {
+        return this.filterBeforeCopyRemote;
+    }
+
+    public void setFilterBeforeCopyRemote(final boolean filterBeforeCopyRemote) {
+        this.filterBeforeCopyRemote = filterBeforeCopyRemote;
+    }
+
+    @Override
+    public void close() throws IOException {
+        cancel();
+        try {
+            join();
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    @Override
+    public void start() {
+        this.future = Utils.runInThread(this::startCopy);
+    }
+
+    @Override
+    public void cancel() {
+        this.lock.lock();
+        try {
+            if (this.cancelled) {
+                return;
+            }
+            if (isOk()) {
+                setError(RaftError.ECANCELED, "Cancel the copier manually.");
+            }
+            this.cancelled = true;
+            if (this.curSession != null) {
+                this.curSession.cancel();
+            }
+            if (this.future != null) {
+                this.future.cancel(true);
+            }
+        } finally {
+            this.lock.unlock();
+        }
+
+    }
+
+    @Override
+    public void join() throws InterruptedException {
+        if (this.future != null) {
+            try {
+                this.future.get();
+            } catch (final InterruptedException e) {
+                throw e;
+            } catch (final CancellationException ignored) {
+                // ignored
+            } catch (final Exception e) {
+                LOG.error("Fail to join on copier", e);
+                throw new IllegalStateException(e);
+            }
+        }
+    }
+
+    @Override
+    public SnapshotReader getReader() {
+        return this.reader;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTable.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTable.java
new file mode 100644
index 0000000..ee99ac9
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTable.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta;
+import com.alipay.sofa.jraft.entity.LocalStorageOutter.LocalSnapshotPbMeta.File;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.io.ProtoBufFile;
+import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.ZeroByteStringHelper;
+
+/**
+ * Table to keep local snapshot metadata infos.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-12 7:22:27 PM
+ */
+public class LocalSnapshotMetaTable {
+
+    private static final Logger              LOG = LoggerFactory.getLogger(LocalSnapshotMetaTable.class);
+
+    private final Map fileMap;
+    private final RaftOptions                raftOptions;
+    private SnapshotMeta                     meta;
+
+    public LocalSnapshotMetaTable(RaftOptions raftOptions) {
+        super();
+        this.fileMap = new HashMap<>();
+        this.raftOptions = raftOptions;
+    }
+
+    /**
+     * Save metadata infos into byte buffer.
+     */
+    public ByteBuffer saveToByteBufferAsRemote() {
+        final LocalSnapshotPbMeta.Builder pbMetaBuilder = LocalSnapshotPbMeta.newBuilder();
+        if (hasMeta()) {
+            pbMetaBuilder.setMeta(this.meta);
+        }
+        for (final Map.Entry entry : this.fileMap.entrySet()) {
+            final File.Builder fb = File.newBuilder() //
+                .setName(entry.getKey()) //
+                .setMeta(entry.getValue());
+            pbMetaBuilder.addFiles(fb.build());
+        }
+        return ByteBuffer.wrap(pbMetaBuilder.build().toByteArray());
+    }
+
+    /**
+     * Load metadata infos from byte buffer.
+     */
+    public boolean loadFromIoBufferAsRemote(final ByteBuffer buf) {
+        if (buf == null) {
+            LOG.error("Null buf to load.");
+            return false;
+        }
+        try {
+            final LocalSnapshotPbMeta pbMeta = LocalSnapshotPbMeta.parseFrom(ZeroByteStringHelper.wrap(buf));
+            if (pbMeta == null) {
+                LOG.error("Fail to load meta from buffer.");
+                return false;
+            }
+            return loadFromPbMeta(pbMeta);
+        } catch (final InvalidProtocolBufferException e) {
+            LOG.error("Fail to parse LocalSnapshotPbMeta from byte buffer", e);
+            return false;
+        }
+    }
+
+    /**
+     * Adds a file metadata.
+     */
+    public boolean addFile(final String fileName, final LocalFileMeta meta) {
+        return this.fileMap.putIfAbsent(fileName, meta) == null;
+    }
+
+    /**
+     * Removes a file metadata.
+     */
+    public boolean removeFile(final String fileName) {
+        return this.fileMap.remove(fileName) != null;
+    }
+
+    /**
+     * Save metadata infos into file by path.
+     */
+    public boolean saveToFile(String path) throws IOException {
+        LocalSnapshotPbMeta.Builder pbMeta = LocalSnapshotPbMeta.newBuilder();
+        if (hasMeta()) {
+            pbMeta.setMeta(this.meta);
+        }
+        for (Map.Entry entry : this.fileMap.entrySet()) {
+            File f = File.newBuilder().setName(entry.getKey()).setMeta(entry.getValue()).build();
+            pbMeta.addFiles(f);
+        }
+        ProtoBufFile pbFile = new ProtoBufFile(path);
+        return pbFile.save(pbMeta.build(), this.raftOptions.isSyncMeta());
+    }
+
+    /**
+     * Returns true when has the snapshot metadata.
+     */
+    public boolean hasMeta() {
+        return this.meta != null && this.meta.isInitialized();
+    }
+
+    /**
+     * Get the file metadata by fileName, returns null when not found.
+     */
+    public LocalFileMeta getFileMeta(String fileName) {
+        return this.fileMap.get(fileName);
+    }
+
+    /**
+     * Get all fileNames in this table.
+     */
+    public Set listFiles() {
+        return this.fileMap.keySet();
+    }
+
+    /**
+     * Set the snapshot metadata.
+     */
+    public void setMeta(SnapshotMeta meta) {
+        this.meta = meta;
+    }
+
+    /**
+     * Returns the snapshot metadata.
+     */
+    public SnapshotMeta getMeta() {
+        return this.meta;
+    }
+
+    /**
+     * Load metadata infos from a file by path.
+     */
+    public boolean loadFromFile(String path) throws IOException {
+        ProtoBufFile pbFile = new ProtoBufFile(path);
+        LocalSnapshotPbMeta pbMeta = pbFile.load();
+        if (pbMeta == null) {
+            LOG.error("Fail to load meta from {}.", path);
+            return false;
+        }
+        return loadFromPbMeta(pbMeta);
+    }
+
+    private boolean loadFromPbMeta(final LocalSnapshotPbMeta pbMeta) {
+        if (pbMeta.hasMeta()) {
+            this.meta = pbMeta.getMeta();
+        } else {
+            this.meta = null;
+        }
+        this.fileMap.clear();
+        for (final File f : pbMeta.getFilesList()) {
+            this.fileMap.put(f.getName(), f.getMeta());
+        }
+        return true;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReader.java
new file mode 100644
index 0000000..0879123
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReader.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.FileService;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.OnlyForTest;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.Message;
+
+/**
+ * Snapshot reader on local file system.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-08 11:10:34 AM
+ */
+public class LocalSnapshotReader extends SnapshotReader {
+
+    private static final Logger          LOG = LoggerFactory.getLogger(LocalSnapshotReader.class);
+
+    /** Generated reader id*/
+    private long                         readerId;
+    /** remote peer addr */
+    private final Endpoint               addr;
+    private final LocalSnapshotMetaTable metaTable;
+    private final String                 path;
+    private final LocalSnapshotStorage   snapshotStorage;
+    private final SnapshotThrottle       snapshotThrottle;
+
+    @Override
+    public void close() throws IOException {
+        snapshotStorage.unref(this.getSnapshotIndex());
+        this.destroyReaderInFileService();
+    }
+
+    public LocalSnapshotReader(LocalSnapshotStorage snapshotStorage, SnapshotThrottle snapshotThrottle, Endpoint addr,
+                               RaftOptions raftOptions, String path) {
+        super();
+        this.snapshotStorage = snapshotStorage;
+        this.snapshotThrottle = snapshotThrottle;
+        this.addr = addr;
+        this.path = path;
+        this.readerId = 0;
+        this.metaTable = new LocalSnapshotMetaTable(raftOptions);
+    }
+
+    @OnlyForTest
+    long getReaderId() {
+        return this.readerId;
+    }
+
+    @Override
+    public boolean init(final Void v) {
+        final File dir = new File(this.path);
+        if (!dir.exists()) {
+            LOG.error("No such path {} for snapshot reader.", this.path);
+            setError(RaftError.ENOENT, "No such path %s for snapshot reader", this.path);
+            return false;
+        }
+        final String metaPath = this.path + File.separator + JRAFT_SNAPSHOT_META_FILE;
+        try {
+            return this.metaTable.loadFromFile(metaPath);
+        } catch (final IOException e) {
+            LOG.error("Fail to load snapshot meta {}.", metaPath, e);
+            setError(RaftError.EIO, "Fail to load snapshot meta from path %s", metaPath);
+            return false;
+        }
+    }
+
+    private long getSnapshotIndex() {
+        final File file = new File(this.path);
+        final String name = file.getName();
+        if (!name.startsWith(JRAFT_SNAPSHOT_PREFIX)) {
+            throw new IllegalStateException("Invalid snapshot path name:" + name);
+        }
+        return Long.parseLong(name.substring(JRAFT_SNAPSHOT_PREFIX.length()));
+    }
+
+    @Override
+    public void shutdown() {
+        Utils.closeQuietly(this);
+    }
+
+    @Override
+    public SnapshotMeta load() {
+        if (this.metaTable.hasMeta()) {
+            return this.metaTable.getMeta();
+        }
+        return null;
+    }
+
+    @Override
+    public String generateURIForCopy() {
+        if (this.addr == null || this.addr.equals(new Endpoint(Utils.IP_ANY, 0))) {
+            LOG.error("Address is not specified");
+            return null;
+        }
+        if (this.readerId == 0) {
+            final SnapshotFileReader reader = new SnapshotFileReader(this.path, this.snapshotThrottle);
+            reader.setMetaTable(this.metaTable);
+            if (!reader.open()) {
+                LOG.error("Open snapshot {} failed.", this.path);
+                return null;
+            }
+            this.readerId = FileService.getInstance().addReader(reader);
+            if (this.readerId < 0) {
+                LOG.error("Fail to add reader to file_service.");
+                return null;
+            }
+        }
+
+        return String.format(REMOTE_SNAPSHOT_URI_SCHEME + "%s/%d", this.addr, this.readerId);
+    }
+
+    private void destroyReaderInFileService() {
+        if (this.readerId > 0) {
+            FileService.getInstance().removeReader(this.readerId);
+            this.readerId = 0;
+        } else {
+            if (this.readerId != 0) {
+                LOG.warn("Ignore destroy invalid readerId: {}", this.readerId);
+            }
+        }
+    }
+
+    @Override
+    public String getPath() {
+        return this.path;
+    }
+
+    @Override
+    public Set listFiles() {
+        return this.metaTable.listFiles();
+    }
+
+    @Override
+    public Message getFileMeta(final String fileName) {
+        return this.metaTable.getFileMeta(fileName);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorage.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorage.java
new file mode 100644
index 0000000..6b0cf33
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorage.java
@@ -0,0 +1,352 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotCopier;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Snapshot storage based on local file storage.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-13 2:11:30 PM
+ */
+public class LocalSnapshotStorage implements SnapshotStorage {
+
+    private static final Logger                      LOG       = LoggerFactory.getLogger(LocalSnapshotStorage.class);
+
+    private static final String                      TEMP_PATH = "temp";
+    private final ConcurrentMap refMap    = new ConcurrentHashMap<>();
+    private final String                             path;
+    private Endpoint                                 addr;
+    private boolean                                  filterBeforeCopyRemote;
+    private long                                     lastSnapshotIndex;
+    private final Lock                               lock;
+    private final RaftOptions                        raftOptions;
+    private SnapshotThrottle                         snapshotThrottle;
+
+    @Override
+    public void setSnapshotThrottle(SnapshotThrottle snapshotThrottle) {
+        this.snapshotThrottle = snapshotThrottle;
+    }
+
+    public boolean hasServerAddr() {
+        return this.addr != null;
+    }
+
+    public void setServerAddr(Endpoint addr) {
+        this.addr = addr;
+    }
+
+    public LocalSnapshotStorage(String path, RaftOptions raftOptions) {
+        super();
+        this.path = path;
+        this.lastSnapshotIndex = 0;
+        this.raftOptions = raftOptions;
+        this.lock = new ReentrantLock();
+    }
+
+    public long getLastSnapshotIndex() {
+        this.lock.lock();
+        try {
+            return this.lastSnapshotIndex;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    @Override
+    public boolean init(final Void v) {
+        final File dir = new File(this.path);
+
+        try {
+            FileUtils.forceMkdir(dir);
+        } catch (final IOException e) {
+            LOG.error("Fail to create directory {}.", this.path, e);
+            return false;
+        }
+
+        // delete temp snapshot
+        if (!this.filterBeforeCopyRemote) {
+            final String tempSnapshotPath = this.path + File.separator + TEMP_PATH;
+            final File tempFile = new File(tempSnapshotPath);
+            if (tempFile.exists()) {
+                try {
+                    FileUtils.forceDelete(tempFile);
+                } catch (final IOException e) {
+                    LOG.error("Fail to delete temp snapshot path {}.", tempSnapshotPath, e);
+                    return false;
+                }
+            }
+        }
+        // delete old snapshot
+        final List snapshots = new ArrayList<>();
+        final File[] oldFiles = dir.listFiles();
+        if (oldFiles != null) {
+            for (final File sFile : oldFiles) {
+                final String name = sFile.getName();
+                if (!name.startsWith(Snapshot.JRAFT_SNAPSHOT_PREFIX)) {
+                    continue;
+                }
+                final long index = Long.parseLong(name.substring(Snapshot.JRAFT_SNAPSHOT_PREFIX.length()));
+                snapshots.add(index);
+            }
+        }
+
+        // TODO: add snapshot watcher
+
+        // get last_snapshot_index
+        if (!snapshots.isEmpty()) {
+            Collections.sort(snapshots);
+            final int snapshotCount = snapshots.size();
+
+            for (int i = 0; i < snapshotCount - 1; i++) {
+                final long index = snapshots.get(i);
+                final String snapshotPath = getSnapshotPath(index);
+                if (!destroySnapshot(snapshotPath)) {
+                    return false;
+                }
+            }
+            this.lastSnapshotIndex = snapshots.get(snapshotCount - 1);
+            ref(this.lastSnapshotIndex);
+        }
+
+        return true;
+    }
+
+    private String getSnapshotPath(final long index) {
+        return this.path + File.separator + Snapshot.JRAFT_SNAPSHOT_PREFIX + index;
+    }
+
+    void ref(final long index) {
+        final AtomicInteger refs = getRefs(index);
+        refs.incrementAndGet();
+    }
+
+    private boolean destroySnapshot(final String path) {
+        LOG.info("Deleting snapshot {}.", path);
+        final File file = new File(path);
+        try {
+            FileUtils.deleteDirectory(file);
+            return true;
+        } catch (final IOException e) {
+            LOG.error("Fail to destroy snapshot {}.", path, e);
+            return false;
+        }
+    }
+
+    void unref(final long index) {
+        final AtomicInteger refs = getRefs(index);
+        if (refs.decrementAndGet() == 0) {
+            if (this.refMap.remove(index, refs)) {
+                destroySnapshot(getSnapshotPath(index));
+            }
+        }
+    }
+
+    AtomicInteger getRefs(final long index) {
+        AtomicInteger refs = this.refMap.get(index);
+        if (refs == null) {
+            refs = new AtomicInteger(0);
+            final AtomicInteger eRefs = this.refMap.putIfAbsent(index, refs);
+            if (eRefs != null) {
+                refs = eRefs;
+            }
+        }
+        return refs;
+    }
+
+    void close(final LocalSnapshotWriter writer, final boolean keepDataOnError) throws IOException {
+        int ret = writer.getCode();
+        // noinspection ConstantConditions
+        do {
+            if (ret != 0) {
+                break;
+            }
+            try {
+                if (!writer.sync()) {
+                    ret = RaftError.EIO.getNumber();
+                    break;
+                }
+            } catch (final IOException e) {
+                LOG.error("Fail to sync writer {}.", writer.getPath(), e);
+                ret = RaftError.EIO.getNumber();
+                break;
+            }
+            final long oldIndex = getLastSnapshotIndex();
+            final long newIndex = writer.getSnapshotIndex();
+            if (oldIndex == newIndex) {
+                ret = RaftError.EEXISTS.getNumber();
+                break;
+            }
+            // rename temp to new
+            final String tempPath = this.path + File.separator + TEMP_PATH;
+            final String newPath = getSnapshotPath(newIndex);
+
+            if (!destroySnapshot(newPath)) {
+                LOG.warn("Delete new snapshot path failed, path is {}.", newPath);
+                ret = RaftError.EIO.getNumber();
+                break;
+            }
+            LOG.info("Renaming {} to {}.", tempPath, newPath);
+            if (!Utils.atomicMoveFile(new File(tempPath), new File(newPath), true)) {
+                LOG.error("Renamed temp snapshot failed, from path {} to path {}.", tempPath, newPath);
+                ret = RaftError.EIO.getNumber();
+                break;
+            }
+            ref(newIndex);
+            this.lock.lock();
+            try {
+                Requires.requireTrue(oldIndex == this.lastSnapshotIndex);
+                this.lastSnapshotIndex = newIndex;
+            } finally {
+                this.lock.unlock();
+            }
+            unref(oldIndex);
+        } while (false);
+        if (ret != 0 && !keepDataOnError) {
+            destroySnapshot(writer.getPath());
+        }
+        if (ret == RaftError.EIO.getNumber()) {
+            throw new IOException();
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        // ignore
+    }
+
+    @Override
+    public boolean setFilterBeforeCopyRemote() {
+        this.filterBeforeCopyRemote = true;
+        return true;
+    }
+
+    @Override
+    public SnapshotWriter create() {
+        return create(true);
+    }
+
+    public SnapshotWriter create(final boolean fromEmpty) {
+        LocalSnapshotWriter writer = null;
+        // noinspection ConstantConditions
+        do {
+            final String snapshotPath = this.path + File.separator + TEMP_PATH;
+            // delete temp
+            // TODO: Notify watcher before deleting
+            if (new File(snapshotPath).exists() && fromEmpty) {
+                if (!destroySnapshot(snapshotPath)) {
+                    break;
+                }
+            }
+            writer = new LocalSnapshotWriter(snapshotPath, this, this.raftOptions);
+            if (!writer.init(null)) {
+                LOG.error("Fail to init snapshot writer.");
+                writer = null;
+                break;
+            }
+        } while (false);
+        return writer;
+    }
+
+    @Override
+    public SnapshotReader open() {
+        long lsIndex = 0;
+        this.lock.lock();
+        try {
+            if (this.lastSnapshotIndex != 0) {
+                lsIndex = this.lastSnapshotIndex;
+                ref(lsIndex);
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        if (lsIndex == 0) {
+            LOG.warn("No data for snapshot reader {}.", this.path);
+            return null;
+        }
+        final String snapshotPath = getSnapshotPath(lsIndex);
+        final SnapshotReader reader = new LocalSnapshotReader(this, this.snapshotThrottle, this.addr, this.raftOptions,
+            snapshotPath);
+        if (!reader.init(null)) {
+            LOG.error("Fail to init reader for path {}.", snapshotPath);
+            unref(lsIndex);
+            return null;
+        }
+        return reader;
+    }
+
+    @Override
+    public SnapshotReader copyFrom(final String uri, final SnapshotCopierOptions opts) {
+        final SnapshotCopier copier = startToCopyFrom(uri, opts);
+        if (copier == null) {
+            return null;
+        }
+        try {
+            copier.join();
+        } catch (final InterruptedException e) {
+            Thread.currentThread().interrupt();
+            LOG.error("Join on snapshot copier was interrupted.");
+            return null;
+        }
+        final SnapshotReader reader = copier.getReader();
+        Utils.closeQuietly(copier);
+        return reader;
+    }
+
+    @Override
+    public SnapshotCopier startToCopyFrom(final String uri, final SnapshotCopierOptions opts) {
+        final LocalSnapshotCopier copier = new LocalSnapshotCopier();
+        copier.setStorage(this);
+        copier.setSnapshotThrottle(this.snapshotThrottle);
+        copier.setFilterBeforeCopyRemote(this.filterBeforeCopyRemote);
+        if (!copier.init(uri, opts)) {
+            LOG.error("Fail to init copier to {}.", uri);
+            return null;
+        }
+        copier.start();
+        return copier;
+    }
+
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriter.java
new file mode 100644
index 0000000..c9a9617
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriter.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta.Builder;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.Message;
+
+/**
+ * Snapshot writer to write snapshot into local file system.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-08 11:51:43 AM
+ */
+public class LocalSnapshotWriter extends SnapshotWriter {
+
+    private static final Logger          LOG = LoggerFactory.getLogger(LocalSnapshotWriter.class);
+
+    private final LocalSnapshotMetaTable metaTable;
+    private final String                 path;
+    private final LocalSnapshotStorage   snapshotStorage;
+
+    public LocalSnapshotWriter(String path, LocalSnapshotStorage snapshotStorage, RaftOptions raftOptions) {
+        super();
+        this.snapshotStorage = snapshotStorage;
+        this.path = path;
+        this.metaTable = new LocalSnapshotMetaTable(raftOptions);
+    }
+
+    @Override
+    public boolean init(final Void v) {
+        final File dir = new File(this.path);
+        try {
+            FileUtils.forceMkdir(dir);
+        } catch (final IOException e) {
+            LOG.error("Fail to create directory {}.", this.path, e);
+            setError(RaftError.EIO, "Fail to create directory  %s", this.path);
+            return false;
+        }
+        final String metaPath = path + File.separator + JRAFT_SNAPSHOT_META_FILE;
+        final File metaFile = new File(metaPath);
+        try {
+            if (metaFile.exists()) {
+                return metaTable.loadFromFile(metaPath);
+            }
+        } catch (final IOException e) {
+            LOG.error("Fail to load snapshot meta from {}.", metaPath, e);
+            setError(RaftError.EIO, "Fail to load snapshot meta from %s", metaPath);
+            return false;
+        }
+        return true;
+    }
+
+    public long getSnapshotIndex() {
+        return this.metaTable.hasMeta() ? this.metaTable.getMeta().getLastIncludedIndex() : 0;
+    }
+
+    @Override
+    public void shutdown() {
+        Utils.closeQuietly(this);
+    }
+
+    @Override
+    public void close() throws IOException {
+        close(false);
+    }
+
+    @Override
+    public void close(final boolean keepDataOnError) throws IOException {
+        this.snapshotStorage.close(this, keepDataOnError);
+    }
+
+    @Override
+    public boolean saveMeta(final SnapshotMeta meta) {
+        this.metaTable.setMeta(meta);
+        return true;
+    }
+
+    public boolean sync() throws IOException {
+        return this.metaTable.saveToFile(this.path + File.separator + JRAFT_SNAPSHOT_META_FILE);
+    }
+
+    @Override
+    public boolean addFile(final String fileName, final Message fileMeta) {
+        final Builder metaBuilder = LocalFileMeta.newBuilder();
+        if (fileMeta != null) {
+            metaBuilder.mergeFrom(fileMeta);
+        }
+        final LocalFileMeta meta = metaBuilder.build();
+        return this.metaTable.addFile(fileName, meta);
+    }
+
+    @Override
+    public boolean removeFile(final String fileName) {
+        return this.metaTable.removeFile(fileName);
+    }
+
+    @Override
+    public String getPath() {
+        return this.path;
+    }
+
+    @Override
+    public Set listFiles() {
+        return this.metaTable.listFiles();
+    }
+
+    @Override
+    public Message getFileMeta(final String fileName) {
+        return this.metaTable.getFileMeta(fileName);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReader.java
new file mode 100644
index 0000000..e2c7e6e
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReader.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.error.RetryAgainException;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.io.LocalDirReader;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+
+/**
+ * Snapshot file reader
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-13 2:03:09 PM
+ */
+public class SnapshotFileReader extends LocalDirReader {
+
+    private final SnapshotThrottle snapshotThrottle;
+    private LocalSnapshotMetaTable metaTable;
+
+    public SnapshotFileReader(String path, SnapshotThrottle snapshotThrottle) {
+        super(path);
+        this.snapshotThrottle = snapshotThrottle;
+    }
+
+    public LocalSnapshotMetaTable getMetaTable() {
+        return this.metaTable;
+    }
+
+    public void setMetaTable(LocalSnapshotMetaTable metaTable) {
+        this.metaTable = metaTable;
+    }
+
+    public boolean open() {
+        final File file = new File(getPath());
+        return file.exists();
+    }
+
+    @Override
+    public int readFile(final ByteBufferCollector metaBufferCollector, final String fileName, final long offset,
+                        final long maxCount) throws IOException, RetryAgainException {
+        // read the whole meta file.
+        if (fileName.equals(Snapshot.JRAFT_SNAPSHOT_META_FILE)) {
+            final ByteBuffer metaBuf = this.metaTable.saveToByteBufferAsRemote();
+            // because bufRef will flip the buffer before using, so we must set the meta buffer position to it's limit.
+            metaBuf.position(metaBuf.limit());
+            metaBufferCollector.setBuffer(metaBuf);
+            return EOF;
+        }
+        final LocalFileMeta fileMeta = this.metaTable.getFileMeta(fileName);
+        if (fileMeta == null) {
+            throw new FileNotFoundException("LocalFileMeta not found for " + fileName);
+        }
+
+        // go through throttle
+        long newMaxCount = maxCount;
+        if (this.snapshotThrottle != null) {
+            newMaxCount = this.snapshotThrottle.throttledByThroughput(maxCount);
+            if (newMaxCount < maxCount) {
+                // if it's not allowed to read partly or it's allowed but
+                // throughput is throttled to 0, try again.
+                if (newMaxCount == 0) {
+                    throw new RetryAgainException("readFile throttled by throughput");
+                }
+            }
+        }
+
+        return readFileWithMeta(metaBufferCollector, fileName, fileMeta, offset, newMaxCount);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySession.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySession.java
new file mode 100644
index 0000000..5e227f9
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySession.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.remote;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import javax.annotation.concurrent.ThreadSafe;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.Scheduler;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.CopyOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileResponse;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter;
+import com.alipay.sofa.jraft.rpc.RpcUtils;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.OnlyForTest;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.Message;
+
+/**
+ * Copy session.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-08 12:01:23 PM
+ */
+@ThreadSafe
+public class CopySession implements Session {
+
+    private static final Logger          LOG         = LoggerFactory.getLogger(CopySession.class);
+
+    private final Lock                   lock        = new ReentrantLock();
+    private final Status                 st          = Status.OK();
+    private final CountDownLatch         finishLatch = new CountDownLatch(1);
+    private final GetFileResponseClosure done        = new GetFileResponseClosure();
+    private final RaftClientService      rpcService;
+    private final GetFileRequest.Builder requestBuilder;
+    private final Endpoint               endpoint;
+    private final Scheduler              timerManager;
+    private final SnapshotThrottle       snapshotThrottle;
+    private final RaftOptions            raftOptions;
+    private int                          retryTimes  = 0;
+    private boolean                      finished;
+    private ByteBufferCollector          destBuf;
+    private CopyOptions                  copyOptions = new CopyOptions();
+    private OutputStream                 outputStream;
+    private ScheduledFuture           timer;
+    private String                       destPath;
+    private Future              rpcCall;
+
+    /**
+     * Get file response closure to answer client.
+     *
+     * @author boyan (boyan@alibaba-inc.com)
+     */
+    private class GetFileResponseClosure extends RpcResponseClosureAdapter {
+
+        @Override
+        public void run(final Status status) {
+            onRpcReturned(status, getResponse());
+        }
+    }
+
+    public void setDestPath(final String destPath) {
+        this.destPath = destPath;
+    }
+
+    @OnlyForTest
+    GetFileResponseClosure getDone() {
+        return this.done;
+    }
+
+    @OnlyForTest
+    Future getRpcCall() {
+        return this.rpcCall;
+    }
+
+    @OnlyForTest
+    ScheduledFuture getTimer() {
+        return this.timer;
+    }
+
+    @Override
+    public void close() throws IOException {
+        this.lock.lock();
+        try {
+            if (!this.finished) {
+                Utils.closeQuietly(this.outputStream);
+            }
+            if (null != this.destBuf) {
+                this.destBuf.recycle();
+                this.destBuf = null;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public CopySession(final RaftClientService rpcService, final Scheduler timerManager,
+                       final SnapshotThrottle snapshotThrottle, final RaftOptions raftOptions,
+                       final GetFileRequest.Builder rb, final Endpoint ep) {
+        super();
+        this.snapshotThrottle = snapshotThrottle;
+        this.raftOptions = raftOptions;
+        this.timerManager = timerManager;
+        this.rpcService = rpcService;
+        this.requestBuilder = rb;
+        this.endpoint = ep;
+    }
+
+    public void setDestBuf(final ByteBufferCollector bufRef) {
+        this.destBuf = bufRef;
+    }
+
+    public void setCopyOptions(final CopyOptions copyOptions) {
+        this.copyOptions = copyOptions;
+    }
+
+    public void setOutputStream(final OutputStream out) {
+        this.outputStream = out;
+    }
+
+    @Override
+    public void cancel() {
+        this.lock.lock();
+        try {
+            if (this.finished) {
+                return;
+            }
+            if (this.timer != null) {
+                this.timer.cancel(true);
+            }
+            if (this.rpcCall != null) {
+                this.rpcCall.cancel(true);
+            }
+            if (this.st.isOk()) {
+                this.st.setError(RaftError.ECANCELED, RaftError.ECANCELED.name());
+            }
+            onFinished();
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    @Override
+    public void join() throws InterruptedException {
+        this.finishLatch.await();
+    }
+
+    @Override
+    public Status status() {
+        return this.st;
+    }
+
+    private void onFinished() {
+        if (!this.finished) {
+            if (!this.st.isOk()) {
+                LOG.error("Fail to copy data, readerId={} fileName={} offset={} status={}",
+                    this.requestBuilder.getReaderId(), this.requestBuilder.getFilename(),
+                    this.requestBuilder.getOffset(), this.st);
+            }
+            if (this.outputStream != null) {
+                Utils.closeQuietly(this.outputStream);
+                this.outputStream = null;
+            }
+            if (this.destBuf != null) {
+                final ByteBuffer buf = this.destBuf.getBuffer();
+                if (buf != null) {
+                    buf.flip();
+                }
+                this.destBuf = null;
+            }
+            this.finished = true;
+            this.finishLatch.countDown();
+        }
+    }
+
+    private void onTimer() {
+        RpcUtils.runInThread(this::sendNextRpc);
+    }
+
+    void onRpcReturned(final Status status, final GetFileResponse response) {
+        this.lock.lock();
+        try {
+            if (this.finished) {
+                return;
+            }
+            if (!status.isOk()) {
+                // Reset count to make next rpc retry the previous one
+                this.requestBuilder.setCount(0);
+                if (status.getCode() == RaftError.ECANCELED.getNumber()) {
+                    if (this.st.isOk()) {
+                        this.st.setError(status.getCode(), status.getErrorMsg());
+                        onFinished();
+                        return;
+                    }
+                }
+
+                // Throttled reading failure does not increase _retry_times
+                if (status.getCode() != RaftError.EAGAIN.getNumber()
+                        && ++this.retryTimes >= this.copyOptions.getMaxRetry()) {
+                    if (this.st.isOk()) {
+                        this.st.setError(status.getCode(), status.getErrorMsg());
+                        onFinished();
+                        return;
+                    }
+                }
+                this.timer = this.timerManager.schedule(this::onTimer, this.copyOptions.getRetryIntervalMs(),
+                    TimeUnit.MILLISECONDS);
+                return;
+            }
+            this.retryTimes = 0;
+            Requires.requireNonNull(response, "response");
+            // Reset count to |real_read_size| to make next rpc get the right offset
+            if (!response.getEof()) {
+                this.requestBuilder.setCount(response.getReadSize());
+            }
+            if (this.outputStream != null) {
+                try {
+                    response.getData().writeTo(this.outputStream);
+                } catch (final IOException e) {
+                    LOG.error("Fail to write into file {}", this.destPath, e);
+                    this.st.setError(RaftError.EIO, RaftError.EIO.name());
+                    onFinished();
+                    return;
+                }
+            } else {
+                this.destBuf.put(response.getData().asReadOnlyByteBuffer());
+            }
+            if (response.getEof()) {
+                onFinished();
+                return;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        sendNextRpc();
+    }
+
+    /**
+     * Send next RPC request to get a piece of file data.
+     */
+    void sendNextRpc() {
+        this.lock.lock();
+        try {
+            this.timer = null;
+            final long offset = this.requestBuilder.getOffset() + this.requestBuilder.getCount();
+            final long maxCount = this.destBuf == null ? this.raftOptions.getMaxByteCountPerRpc() : Integer.MAX_VALUE;
+            this.requestBuilder.setOffset(offset).setCount(maxCount).setReadPartly(true);
+
+            if (this.finished) {
+                return;
+            }
+            // throttle
+            long newMaxCount = maxCount;
+            if (this.snapshotThrottle != null) {
+                newMaxCount = this.snapshotThrottle.throttledByThroughput(maxCount);
+                if (newMaxCount == 0) {
+                    // Reset count to make next rpc retry the previous one
+                    this.requestBuilder.setCount(0);
+                    this.timer = this.timerManager.schedule(this::onTimer, this.copyOptions.getRetryIntervalMs(),
+                        TimeUnit.MILLISECONDS);
+                    return;
+                }
+            }
+            this.requestBuilder.setCount(newMaxCount);
+            final RpcRequests.GetFileRequest request = this.requestBuilder.build();
+            LOG.debug("Send get file request {} to peer {}", request, this.endpoint);
+            this.rpcCall = this.rpcService.getFile(this.endpoint, request, this.copyOptions.getTimeoutMs(), this.done);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopier.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopier.java
new file mode 100644
index 0000000..b8614e2
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopier.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.remote;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.core.Scheduler;
+import com.alipay.sofa.jraft.option.CopyOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.GetFileRequest;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.OnlyForTest;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Remote file copier
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-23 2:03:14 PM
+ */
+public class RemoteFileCopier {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RemoteFileCopier.class);
+
+    private long                readId;
+    private RaftClientService   rpcService;
+    private Endpoint            endpoint;
+    private RaftOptions         raftOptions;
+    private Scheduler           timerManager;
+    private SnapshotThrottle    snapshotThrottle;
+
+    @OnlyForTest
+    long getReaderId() {
+        return this.readId;
+    }
+
+    @OnlyForTest
+    Endpoint getEndpoint() {
+        return this.endpoint;
+    }
+
+    public boolean init(String uri, final SnapshotThrottle snapshotThrottle, final SnapshotCopierOptions opts) {
+        this.rpcService = opts.getRaftClientService();
+        this.timerManager = opts.getTimerManager();
+        this.raftOptions = opts.getRaftOptions();
+        this.snapshotThrottle = snapshotThrottle;
+
+        final int prefixSize = Snapshot.REMOTE_SNAPSHOT_URI_SCHEME.length();
+        if (uri == null || !uri.startsWith(Snapshot.REMOTE_SNAPSHOT_URI_SCHEME)) {
+            LOG.error("Invalid uri {}.", uri);
+            return false;
+        }
+        uri = uri.substring(prefixSize);
+        final int slasPos = uri.indexOf('/');
+        final String ipAndPort = uri.substring(0, slasPos);
+        uri = uri.substring(slasPos + 1);
+
+        try {
+            this.readId = Long.parseLong(uri);
+            final String[] ipAndPortStrs = ipAndPort.split(":");
+            this.endpoint = new Endpoint(ipAndPortStrs[0], Integer.parseInt(ipAndPortStrs[1]));
+        } catch (final Exception e) {
+            LOG.error("Fail to parse readerId or endpoint.", e);
+            return false;
+        }
+        if (!this.rpcService.connect(this.endpoint)) {
+            LOG.error("Fail to init channel to {}.", this.endpoint);
+            return false;
+        }
+
+        return true;
+    }
+
+    /**
+     * Copy `source` from remote to local dest.
+     *
+     * @param source   source from remote
+     * @param destPath local path
+     * @param opts     options of copy
+     * @return true if copy success
+     */
+    public boolean copyToFile(final String source, final String destPath, final CopyOptions opts) throws IOException,
+                                                                                                 InterruptedException {
+        final Session session = startCopyToFile(source, destPath, opts);
+        if (session == null) {
+            return false;
+        }
+        try {
+            session.join();
+            return session.status().isOk();
+        } finally {
+            Utils.closeQuietly(session);
+        }
+    }
+
+    public Session startCopyToFile(final String source, final String destPath, final CopyOptions opts)
+                                                                                                      throws IOException {
+        final File file = new File(destPath);
+
+        // delete exists file.
+        if (file.exists()) {
+            if (!file.delete()) {
+                LOG.error("Fail to delete destPath: {}.", destPath);
+                return null;
+            }
+        }
+
+        final OutputStream out = new BufferedOutputStream(new FileOutputStream(file, false) {
+
+            @Override
+            public void close() throws IOException {
+                getFD().sync();
+                super.close();
+            }
+        });
+        final CopySession session = newCopySession(source);
+        session.setOutputStream(out);
+        session.setDestPath(destPath);
+        session.setDestBuf(null);
+        if (opts != null) {
+            session.setCopyOptions(opts);
+        }
+        session.sendNextRpc();
+        return session;
+    }
+
+    private CopySession newCopySession(final String source) {
+        final GetFileRequest.Builder reqBuilder = GetFileRequest.newBuilder() //
+            .setFilename(source) //
+            .setReaderId(this.readId);
+        return new CopySession(this.rpcService, this.timerManager, this.snapshotThrottle, this.raftOptions, reqBuilder,
+            this.endpoint);
+    }
+
+    /**
+     * Copy `source` from remote to  buffer.
+     * @param source  source from remote
+     * @param destBuf buffer of dest
+     * @param opt     options of copy
+     * @return true if copy success
+     */
+    public boolean copy2IoBuffer(final String source, final ByteBufferCollector destBuf, final CopyOptions opt)
+                                                                                                               throws InterruptedException {
+        final Session session = startCopy2IoBuffer(source, destBuf, opt);
+        if (session == null) {
+            return false;
+        }
+        try {
+            session.join();
+            return session.status().isOk();
+        } finally {
+            Utils.closeQuietly(session);
+        }
+    }
+
+    public Session startCopy2IoBuffer(final String source, final ByteBufferCollector destBuf, final CopyOptions opts) {
+        final CopySession session = newCopySession(source);
+        session.setOutputStream(null);
+        session.setDestBuf(destBuf);
+        if (opts != null) {
+            session.setCopyOptions(opts);
+        }
+        session.sendNextRpc();
+        return session;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/Session.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/Session.java
new file mode 100644
index 0000000..176971a
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/storage/snapshot/remote/Session.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.remote;
+
+import java.io.Closeable;
+
+import com.alipay.sofa.jraft.Status;
+
+/**
+ * A copy session.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-08 12:02:12 PM
+ */
+public interface Session extends Closeable {
+
+    /**
+     * Cancel the copy job.
+     */
+    void cancel();
+
+    /**
+     * Block the thread to wait the copy job finishes or canceled.
+     *
+     * @throws InterruptedException if the current thread is interrupted
+     *         while waiting
+     */
+    void join() throws InterruptedException;
+
+    /**
+     * Returns the copy job status.
+     */
+    Status status();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocator.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocator.java
new file mode 100644
index 0000000..8414d1c
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocator.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The code comes from https://github.com/netty/netty/blob/4.1/transport/src/main/java/io/netty/channel/AdaptiveRecvByteBufAllocator.java
+ *
+ * @author jiachun.fjc
+ */
+public class AdaptiveBufAllocator {
+
+    private static final int                 DEFAULT_MINIMUM = 64;
+    private static final int                 DEFAULT_INITIAL = 512;
+    private static final int                 DEFAULT_MAXIMUM = 524288;
+
+    private static final int                 INDEX_INCREMENT = 4;
+    private static final int                 INDEX_DECREMENT = 1;
+
+    private static final int[]               SIZE_TABLE;
+
+    static {
+        final List sizeTable = new ArrayList<>();
+        for (int i = 16; i < 512; i += 16) {
+            sizeTable.add(i);
+        }
+
+        for (int i = 512; i > 0; i <<= 1) {
+            sizeTable.add(i);
+        }
+
+        SIZE_TABLE = new int[sizeTable.size()];
+        for (int i = 0; i < SIZE_TABLE.length; i++) {
+            SIZE_TABLE[i] = sizeTable.get(i);
+        }
+    }
+
+    public static final AdaptiveBufAllocator DEFAULT         = new AdaptiveBufAllocator();
+
+    private static int getSizeTableIndex(final int size) {
+        for (int low = 0, high = SIZE_TABLE.length - 1;;) {
+            if (high < low) {
+                return low;
+            }
+            if (high == low) {
+                return high;
+            }
+
+            final int mid = low + high >>> 1;
+            final int a = SIZE_TABLE[mid];
+            final int b = SIZE_TABLE[mid + 1];
+            if (size > b) {
+                low = mid + 1;
+            } else if (size < a) {
+                high = mid - 1;
+            } else if (size == a) {
+                return mid;
+            } else {
+                return mid + 1;
+            }
+        }
+    }
+
+    public interface Handle {
+
+        /**
+         * Creates a new buffer whose capacity is probably large enough to write all outbound data and small
+         * enough not to waste its space.
+         */
+        ByteBufferCollector allocate();
+
+        /**
+         * Gets a buffer from recyclers whose capacity is probably large enough to write all outbound data and
+         * small enough not to waste its space, recycling is needed.
+         */
+        ByteBufferCollector allocateByRecyclers();
+
+        /**
+         * Similar to {@link #allocate()} except that it does not allocate anything but
+         * just tells the capacity.
+         */
+        int guess();
+
+        /**
+         * Records the the actual number of wrote bytes in the previous write operation so that the allocator
+         * allocates the buffer with potentially more correct capacity.
+         *
+         * @param actualWroteBytes the actual number of wrote bytes in the previous allocate operation
+         */
+        void record(final int actualWroteBytes);
+    }
+
+    private static final class HandleImpl implements Handle {
+
+        private final int minIndex;
+        private final int maxIndex;
+        private int       index;
+        private int       nextAllocateBufSize;
+        private boolean   decreaseNow;
+
+        HandleImpl(int minIndex, int maxIndex, int initial) {
+            this.minIndex = minIndex;
+            this.maxIndex = maxIndex;
+
+            this.index = getSizeTableIndex(initial);
+            this.nextAllocateBufSize = SIZE_TABLE[this.index];
+        }
+
+        @Override
+        public ByteBufferCollector allocate() {
+            return ByteBufferCollector.allocate(guess());
+        }
+
+        @Override
+        public ByteBufferCollector allocateByRecyclers() {
+            return ByteBufferCollector.allocateByRecyclers(guess());
+        }
+
+        @Override
+        public int guess() {
+            return this.nextAllocateBufSize;
+        }
+
+        @Override
+        public void record(final int actualWroteBytes) {
+            if (actualWroteBytes <= SIZE_TABLE[Math.max(0, this.index - INDEX_DECREMENT - 1)]) {
+                if (this.decreaseNow) {
+                    this.index = Math.max(this.index - INDEX_DECREMENT, this.minIndex);
+                    this.nextAllocateBufSize = SIZE_TABLE[this.index];
+                    this.decreaseNow = false;
+                } else {
+                    this.decreaseNow = true;
+                }
+            } else if (actualWroteBytes >= this.nextAllocateBufSize) {
+                this.index = Math.min(this.index + INDEX_INCREMENT, this.maxIndex);
+                this.nextAllocateBufSize = SIZE_TABLE[this.index];
+                this.decreaseNow = false;
+            }
+        }
+    }
+
+    private final int minIndex;
+    private final int maxIndex;
+    private final int initial;
+
+    /**
+     * Creates a new predictor with the default parameters.  With the default
+     * parameters, the expected buffer size starts from {@code 512}, does not
+     * go down below {@code 64}, and does not go up above {@code 524288}.
+     */
+    private AdaptiveBufAllocator() {
+        this(DEFAULT_MINIMUM, DEFAULT_INITIAL, DEFAULT_MAXIMUM);
+    }
+
+    /**
+     * Creates a new predictor with the specified parameters.
+     *
+     * @param minimum the inclusive lower bound of the expected buffer size
+     * @param initial the initial buffer size when no feed back was received
+     * @param maximum the inclusive upper bound of the expected buffer size
+     */
+    public AdaptiveBufAllocator(int minimum, int initial, int maximum) {
+        Requires.requireTrue(minimum > 0, "minimum: " + minimum);
+        Requires.requireTrue(initial >= minimum, "initial: " + initial);
+        Requires.requireTrue(initial <= maximum, "maximum: " + maximum);
+
+        final int minIndex = getSizeTableIndex(minimum);
+        if (SIZE_TABLE[minIndex] < minimum) {
+            this.minIndex = minIndex + 1;
+        } else {
+            this.minIndex = minIndex;
+        }
+
+        final int maxIndex = getSizeTableIndex(maximum);
+        if (SIZE_TABLE[maxIndex] > maximum) {
+            this.maxIndex = maxIndex - 1;
+        } else {
+            this.maxIndex = maxIndex;
+        }
+
+        this.initial = initial;
+    }
+
+    public Handle newHandle() {
+        return new AdaptiveBufAllocator.HandleImpl(this.minIndex, this.maxIndex, this.initial);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ArrayDeque.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ArrayDeque.java
new file mode 100644
index 0000000..d281844
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ArrayDeque.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.List;
+
+/**
+ * Extend array list to add peek/poll first/last element.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-11 11:14:38 AM 
+ * @param 
+ */
+public class ArrayDeque extends java.util.ArrayList {
+
+    private static final long serialVersionUID = -4929318149975955629L;
+
+    /**
+     * Get the first element of list.
+     */
+    public static  E peekFirst(List list) {
+        return list.get(0);
+    }
+
+    /**
+     * Remove the first element from list and return it.
+     */
+    public static  E pollFirst(List list) {
+        return list.remove(0);
+    }
+
+    /**
+     * Get the last element of list.
+     */
+    public static  E peekLast(List list) {
+        return list.get(list.size() - 1);
+    }
+
+    /**
+     * Remove the last element from list and return it.
+     */
+    public static  E pollLast(List list) {
+        return list.remove(list.size() - 1);
+    }
+
+    /**
+     * Get the first element of list.
+     */
+    public E peekFirst() {
+        return peekFirst(this);
+    }
+
+    /**
+     * Get the last element of list.
+     */
+    public E peekLast() {
+        return peekLast(this);
+    }
+
+    /**
+     * Remove the first element from list and return it.
+     */
+    public E pollFirst() {
+        return pollFirst(this);
+    }
+
+    /**
+     * Remove the last element from list and return it.
+     */
+    public E pollLast() {
+        return pollLast(this);
+    }
+
+    /**
+     * Expose this methods so we not need to create a new subList just to
+     * remove a range of elements.
+     *
+     * Removes from this deque all of the elements whose index is between
+     * {@code fromIndex}, inclusive, and {@code toIndex}, exclusive.
+     * Shifts any succeeding elements to the left (reduces their index).
+     * This call shortens the deque by {@code (toIndex - fromIndex)} elements.
+     * (If {@code toIndex==fromIndex}, this operation has no effect.)
+     *
+     * @throws IndexOutOfBoundsException if {@code fromIndex} or
+     *         {@code toIndex} is out of range
+     *         ({@code fromIndex < 0 ||
+     *          fromIndex >= size() ||
+     *          toIndex > size() ||
+     *          toIndex < fromIndex})
+     */
+    @Override
+    public void removeRange(int fromIndex, int toIndex) {
+        super.removeRange(fromIndex, toIndex);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AsciiStringUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AsciiStringUtil.java
new file mode 100644
index 0000000..42e7ade
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/AsciiStringUtil.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+import com.google.protobuf.ByteString;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class AsciiStringUtil {
+
+    public static byte[] unsafeEncode(final CharSequence in) {
+        final int len = in.length();
+        final byte[] out = new byte[len];
+        for (int i = 0; i < len; i++) {
+            out[i] = (byte) in.charAt(i);
+        }
+        return out;
+    }
+
+    public static String unsafeDecode(final byte[] in, final int offset, final int len) {
+        final char[] out = new char[len];
+        for (int i = 0; i < len; i++) {
+            out[i] = (char) (in[i + offset] & 0xFF);
+        }
+        return UnsafeUtil.moveToString(out);
+    }
+
+    public static String unsafeDecode(final byte[] in) {
+        return unsafeDecode(in, 0, in.length);
+    }
+
+    public static String unsafeDecode(final ByteString in) {
+        final int len = in.size();
+        final char[] out = new char[len];
+        for (int i = 0; i < len; i++) {
+            out[i] = (char) (in.byteAt(i) & 0xFF);
+        }
+        return UnsafeUtil.moveToString(out);
+    }
+
+    private AsciiStringUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bits.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bits.java
new file mode 100644
index 0000000..b2947db
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bits.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Bits util.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ * @author jiachun.fjc
+ */
+public class Bits {
+
+    public static int getInt(final byte[] b, final int off) {
+        return HeapByteBufUtil.getInt(b, off);
+    }
+
+    public static long getLong(final byte[] b, final int off) {
+        return HeapByteBufUtil.getLong(b, off);
+    }
+
+    public static void putInt(final byte[] b, final int off, final int val) {
+        HeapByteBufUtil.setInt(b, off, val);
+    }
+
+    public static void putShort(final byte[] b, final int off, final short val) {
+        HeapByteBufUtil.setShort(b, off, val);
+    }
+
+    public static short getShort(final byte[] b, final int off) {
+        return HeapByteBufUtil.getShort(b, off);
+    }
+
+    public static void putLong(final byte[] b, final int off, final long val) {
+        HeapByteBufUtil.setLong(b, off, val);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ByteBufferCollector.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ByteBufferCollector.java
new file mode 100644
index 0000000..96b1f63
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ByteBufferCollector.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * A byte buffer collector that will expand automatically.
+ *
+ * @author dennis
+ */
+public final class ByteBufferCollector implements Recyclable {
+
+    private static final int MAX_CAPACITY_TO_RECYCLE = 4 * 1024 * 1024; // 4M
+
+    private ByteBuffer       buffer;
+
+    public int capacity() {
+        return this.buffer != null ? this.buffer.capacity() : 0;
+    }
+
+    public void expandIfNecessary() {
+        if (!hasRemaining()) {
+            getBuffer(Utils.RAFT_DATA_BUF_SIZE);
+        }
+    }
+
+    public void expandAtMost(final int atMostBytes) {
+        if (this.buffer == null) {
+            this.buffer = Utils.allocate(atMostBytes);
+        } else {
+            this.buffer = Utils.expandByteBufferAtMost(this.buffer, atMostBytes);
+        }
+    }
+
+    public boolean hasRemaining() {
+        return this.buffer != null && this.buffer.hasRemaining();
+    }
+
+    private ByteBufferCollector(final int size, final Recyclers.Handle handle) {
+        if (size > 0) {
+            this.buffer = Utils.allocate(size);
+        }
+        this.handle = handle;
+    }
+
+    public static ByteBufferCollector allocate(final int size) {
+        return new ByteBufferCollector(size, Recyclers.NOOP_HANDLE);
+    }
+
+    public static ByteBufferCollector allocate() {
+        return allocate(Utils.RAFT_DATA_BUF_SIZE);
+    }
+
+    public static ByteBufferCollector allocateByRecyclers(final int size) {
+        final ByteBufferCollector collector = recyclers.get();
+        collector.reset(size);
+        return collector;
+    }
+
+    public static ByteBufferCollector allocateByRecyclers() {
+        return allocateByRecyclers(Utils.RAFT_DATA_BUF_SIZE);
+    }
+
+    public static int threadLocalCapacity() {
+        return recyclers.threadLocalCapacity();
+    }
+
+    public static int threadLocalSize() {
+        return recyclers.threadLocalSize();
+    }
+
+    private void reset(final int expectSize) {
+        if (this.buffer == null) {
+            this.buffer = Utils.allocate(expectSize);
+        } else {
+            if (this.buffer.capacity() < expectSize) {
+                this.buffer = Utils.allocate(expectSize);
+            }
+        }
+    }
+
+    private ByteBuffer getBuffer(final int expectSize) {
+        if (this.buffer == null) {
+            this.buffer = Utils.allocate(expectSize);
+        } else if (this.buffer.remaining() < expectSize) {
+            this.buffer = Utils.expandByteBufferAtLeast(this.buffer, expectSize);
+        }
+        return this.buffer;
+    }
+
+    public void put(final ByteBuffer buf) {
+        getBuffer(buf.remaining()).put(buf);
+    }
+
+    public void put(final byte[] bs) {
+        getBuffer(bs.length).put(bs);
+    }
+
+    public void setBuffer(final ByteBuffer buffer) {
+        this.buffer = buffer;
+    }
+
+    public ByteBuffer getBuffer() {
+        return this.buffer;
+    }
+
+    @Override
+    public boolean recycle() {
+        if (this.buffer != null) {
+            if (this.buffer.capacity() > MAX_CAPACITY_TO_RECYCLE) {
+                // If the size is too large, we should release it to avoid memory overhead
+                this.buffer = null;
+            } else {
+                this.buffer.clear();
+            }
+        }
+        return recyclers.recycle(this, handle);
+    }
+
+    private transient final Recyclers.Handle            handle;
+
+    private static final Recyclers recyclers = new Recyclers(
+                                                                      Utils.MAX_COLLECTOR_SIZE_PER_THREAD) {
+
+                                                                      @Override
+                                                                      protected ByteBufferCollector newObject(final Handle handle) {
+                                                                          return new ByteBufferCollector(0, handle);
+                                                                      }
+                                                                  };
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bytes.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bytes.java
new file mode 100644
index 0000000..8ae5f66
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Bytes.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.Arrays;
+
+/**
+ * Utility class that handles immutable byte arrays.
+ */
+public class Bytes implements Comparable {
+
+    private static final char[] HEX_CHARS_UPPER = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C',
+            'D', 'E', 'F'                      };
+
+    private final byte[]        bytes;
+
+    // cache the hash code for the string, default to 0
+    private int                 hashCode;
+
+    public static Bytes wrap(byte[] bytes) {
+        return new Bytes(bytes);
+    }
+
+    /**
+     * Create a Bytes using the byte array.
+     *
+     * @param bytes This array becomes the backing storage for the object.
+     */
+    public Bytes(byte[] bytes) {
+        this.bytes = bytes;
+
+        // initialize hash code to 0
+        hashCode = 0;
+    }
+
+    /**
+     * Get the data from the Bytes.
+     * @return The underlying byte array
+     */
+    public byte[] get() {
+        return this.bytes;
+    }
+
+    /**
+     * The hashcode is cached except for the case where it is computed as 0, in which
+     * case we compute the hashcode on every call.
+     *
+     * @return the hashcode
+     */
+    @Override
+    public int hashCode() {
+        if (hashCode == 0) {
+            hashCode = Arrays.hashCode(bytes);
+        }
+
+        return hashCode;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (this == other) {
+            return true;
+        }
+        if (other == null) {
+            return false;
+        }
+
+        // we intentionally use the function to compute hashcode here
+        return this.hashCode() == other.hashCode() && other instanceof Bytes
+               && Arrays.equals(this.bytes, ((Bytes) other).get());
+    }
+
+    @Override
+    public int compareTo(Bytes that) {
+        return BytesUtil.getDefaultByteArrayComparator().compare(this.bytes, that.bytes);
+    }
+
+    @Override
+    public String toString() {
+        return Bytes.toString(bytes, 0, bytes.length);
+    }
+
+    /**
+     * Write a printable representation of a byte array. Non-printable
+     * characters are hex escaped in the format \\x%02X, eg:
+     * \x00 \x05 etc.
+     *
+     * This function is brought from org.apache.hadoop.hbase.util.Bytes
+     *
+     * @param b array to write out
+     * @param off offset to start at
+     * @param len length to write
+     * @return string output
+     */
+    @SuppressWarnings("SameParameterValue")
+    private static String toString(final byte[] b, int off, int len) {
+        final StringBuilder result = new StringBuilder();
+
+        if (b == null) {
+            return result.toString();
+        }
+
+        // just in case we are passed a 'len' that is > buffer length...
+        if (off >= b.length) {
+            return result.toString();
+        }
+
+        if (off + len > b.length) {
+            len = b.length - off;
+        }
+
+        for (int i = off; i < off + len; ++i) {
+            final int ch = b[i] & 0xFF;
+            if (ch >= ' ' && ch <= '~' && ch != '\\') {
+                result.append((char) ch);
+            } else {
+                result.append("\\x");
+                result.append(HEX_CHARS_UPPER[ch / 0x10]);
+                result.append(HEX_CHARS_UPPER[ch % 0x10]);
+            }
+        }
+        return result.toString();
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/BytesUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/BytesUtil.java
new file mode 100644
index 0000000..f3364ee
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/BytesUtil.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.Comparator;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtf8Util;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class BytesUtil {
+
+    public static final byte[]               EMPTY_BYTES             = new byte[0];
+
+    // A byte array comparator based on lexicograpic ordering.
+    private static final ByteArrayComparator BYTES_LEXICO_COMPARATOR = new LexicographicByteArrayComparator();
+
+    public static byte[] nullToEmpty(final byte[] bytes) {
+        return bytes == null ? EMPTY_BYTES : bytes;
+    }
+
+    public static boolean isEmpty(final byte[] bytes) {
+        return bytes == null || bytes.length == 0;
+    }
+
+    /**
+     * This method has better performance than String#getBytes(Charset),
+     * See the benchmark class: Utf8Benchmark for details.
+     */
+    public static byte[] writeUtf8(final String in) {
+        if (in == null) {
+            return null;
+        }
+        if (UnsafeUtil.hasUnsafe()) {
+            // Calculate the encoded length.
+            final int len = UnsafeUtf8Util.encodedLength(in);
+            final byte[] outBytes = new byte[len];
+            UnsafeUtf8Util.encodeUtf8(in, outBytes, 0, len);
+            return outBytes;
+        } else {
+            return in.getBytes(StandardCharsets.UTF_8);
+        }
+    }
+
+    /**
+     * This method has better performance than String#String(byte[], Charset),
+     * See the benchmark class: Utf8Benchmark for details.
+     */
+    public static String readUtf8(final byte[] in) {
+        if (in == null) {
+            return null;
+        }
+        if (UnsafeUtil.hasUnsafe()) {
+            return UnsafeUtf8Util.decodeUtf8(in, 0, in.length);
+        } else {
+            return new String(in, StandardCharsets.UTF_8);
+        }
+    }
+
+    public static byte[] nextBytes(final byte[] bytes) {
+        Requires.requireNonNull(bytes, "bytes");
+        final int len = bytes.length;
+        if (len == 0) { // fast path
+            return new byte[] { 0 };
+        }
+        final byte[] nextBytes = new byte[len + 1];
+        System.arraycopy(bytes, 0, nextBytes, 0, len);
+        nextBytes[len] = 0;
+        return nextBytes;
+    }
+
+    public static ByteArrayComparator getDefaultByteArrayComparator() {
+        return BYTES_LEXICO_COMPARATOR;
+    }
+
+    public static int compare(final byte[] a, final byte[] b) {
+        return getDefaultByteArrayComparator().compare(a, b);
+    }
+
+    public static byte[] max(final byte[] a, final byte[] b) {
+        return getDefaultByteArrayComparator().compare(a, b) > 0 ? a : b;
+    }
+
+    public static byte[] min(final byte[] a, final byte[] b) {
+        return getDefaultByteArrayComparator().compare(a, b) < 0 ? a : b;
+    }
+
+    public interface ByteArrayComparator extends Comparator, Serializable {
+
+        int compare(final byte[] buffer1, final int offset1, final int length1, final byte[] buffer2,
+                    final int offset2, final int length2);
+    }
+
+    private static class LexicographicByteArrayComparator implements ByteArrayComparator {
+
+        private static final long serialVersionUID = -8623342242397267864L;
+
+        @Override
+        public int compare(final byte[] buffer1, final byte[] buffer2) {
+            return compare(buffer1, 0, buffer1.length, buffer2, 0, buffer2.length);
+        }
+
+        @Override
+        public int compare(final byte[] buffer1, final int offset1, final int length1, final byte[] buffer2,
+                           final int offset2, final int length2) {
+            // short circuit equal case
+            if (buffer1 == buffer2 && offset1 == offset2 && length1 == length2) {
+                return 0;
+            }
+            // similar to Arrays.compare() but considers offset and length
+            final int end1 = offset1 + length1;
+            final int end2 = offset2 + length2;
+            for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) {
+                int a = buffer1[i] & 0xff;
+                int b = buffer2[j] & 0xff;
+                if (a != b) {
+                    return a - b;
+                }
+            }
+            return length1 - length2;
+        }
+    }
+
+    private final static char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
+
+    /**
+     * Dump byte array into a hex string.
+     * See https://stackoverflow.com/questions/9655181/how-to-convert-a-byte-array-to-a-hex-string-in-java
+     * @param bytes bytes
+     * @return hex string
+     */
+    public static String toHex(final byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
+        final char[] hexChars = new char[bytes.length * 2];
+        for (int j = 0; j < bytes.length; j++) {
+            int v = bytes[j] & 0xFF;
+            hexChars[j * 2] = HEX_ARRAY[v >>> 4];
+            hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
+        }
+        return new String(hexChars);
+    }
+
+    /**
+     * Convert a string representation of a hex dump to a byte array.
+     * See https://stackoverflow.com/questions/140131/convert-a-string-representation-of-a-hex-dump-to-a-byte-array-using-java
+     * @param s hex string
+     * @return bytes
+     */
+    public static byte[] hexStringToByteArray(final String s) {
+        if (s == null) {
+            return null;
+        }
+        final int len = s.length();
+        final byte[] bytes = new byte[len / 2];
+        for (int i = 0; i < len; i += 2) {
+            bytes[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16));
+        }
+        return bytes;
+    }
+
+    private BytesUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CRC64.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CRC64.java
new file mode 100644
index 0000000..5cd61d8
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CRC64.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.zip.Checksum;
+
+/**
+ * CRC64
+ * @author boyan(boyan@antfin.com)
+ *
+ */
+public class CRC64 implements Checksum {
+    /**
+     * CRC-64-ECMA-182
+     *
+     * 
+     * The polynomial code used is 0x42F0E1EBA9EA3693 (CRC64-ECMA-182)
+     * x64+x62+x57+x55+x54+x53+x52+x47+x46+x45+x40+x39+x38+x37+x35+x33+
+     * x32+x31+x29+x27+x24+x23+x22+x21+x19+x17+x13+x12+x10+x9+x7+x4+x+1
+     * 

+     *
+     * poly=0x42f0e1eba9ea3693 init=0x0 refin=false refout=false xorout=0x0
+     *
+     * @see http://en.wikipedia.org/wiki/Cyclic_redundancy_check
+     * @see http://reveng.sourceforge.net/crc-catalogue/17plus.htm
+     */
+    private static final long[] CRC_TABLE = new long[] { 0x0000000000000000L, 0x42F0E1EBA9EA3693L, 0x85E1C3D753D46D26L,
+            0xC711223CFA3E5BB5L, 0x493366450E42ECDFL, 0x0BC387AEA7A8DA4CL, 0xCCD2A5925D9681F9L, 0x8E224479F47CB76AL,
+            0x9266CC8A1C85D9BEL, 0xD0962D61B56FEF2DL, 0x17870F5D4F51B498L, 0x5577EEB6E6BB820BL, 0xDB55AACF12C73561L,
+            0x99A54B24BB2D03F2L, 0x5EB4691841135847L, 0x1C4488F3E8F96ED4L, 0x663D78FF90E185EFL, 0x24CD9914390BB37CL,
+            0xE3DCBB28C335E8C9L, 0xA12C5AC36ADFDE5AL, 0x2F0E1EBA9EA36930L, 0x6DFEFF5137495FA3L, 0xAAEFDD6DCD770416L,
+            0xE81F3C86649D3285L, 0xF45BB4758C645C51L, 0xB6AB559E258E6AC2L, 0x71BA77A2DFB03177L, 0x334A9649765A07E4L,
+            0xBD68D2308226B08EL, 0xFF9833DB2BCC861DL, 0x388911E7D1F2DDA8L, 0x7A79F00C7818EB3BL, 0xCC7AF1FF21C30BDEL,
+            0x8E8A101488293D4DL, 0x499B3228721766F8L, 0x0B6BD3C3DBFD506BL, 0x854997BA2F81E701L, 0xC7B97651866BD192L,
+            0x00A8546D7C558A27L, 0x4258B586D5BFBCB4L, 0x5E1C3D753D46D260L, 0x1CECDC9E94ACE4F3L, 0xDBFDFEA26E92BF46L,
+            0x990D1F49C77889D5L, 0x172F5B3033043EBFL, 0x55DFBADB9AEE082CL, 0x92CE98E760D05399L, 0xD03E790CC93A650AL,
+            0xAA478900B1228E31L, 0xE8B768EB18C8B8A2L, 0x2FA64AD7E2F6E317L, 0x6D56AB3C4B1CD584L, 0xE374EF45BF6062EEL,
+            0xA1840EAE168A547DL, 0x66952C92ECB40FC8L, 0x2465CD79455E395BL, 0x3821458AADA7578FL, 0x7AD1A461044D611CL,
+            0xBDC0865DFE733AA9L, 0xFF3067B657990C3AL, 0x711223CFA3E5BB50L, 0x33E2C2240A0F8DC3L, 0xF4F3E018F031D676L,
+            0xB60301F359DBE0E5L, 0xDA050215EA6C212FL, 0x98F5E3FE438617BCL, 0x5FE4C1C2B9B84C09L, 0x1D14202910527A9AL,
+            0x93366450E42ECDF0L, 0xD1C685BB4DC4FB63L, 0x16D7A787B7FAA0D6L, 0x5427466C1E109645L, 0x4863CE9FF6E9F891L,
+            0x0A932F745F03CE02L, 0xCD820D48A53D95B7L, 0x8F72ECA30CD7A324L, 0x0150A8DAF8AB144EL, 0x43A04931514122DDL,
+            0x84B16B0DAB7F7968L, 0xC6418AE602954FFBL, 0xBC387AEA7A8DA4C0L, 0xFEC89B01D3679253L, 0x39D9B93D2959C9E6L,
+            0x7B2958D680B3FF75L, 0xF50B1CAF74CF481FL, 0xB7FBFD44DD257E8CL, 0x70EADF78271B2539L, 0x321A3E938EF113AAL,
+            0x2E5EB66066087D7EL, 0x6CAE578BCFE24BEDL, 0xABBF75B735DC1058L, 0xE94F945C9C3626CBL, 0x676DD025684A91A1L,
+            0x259D31CEC1A0A732L, 0xE28C13F23B9EFC87L, 0xA07CF2199274CA14L, 0x167FF3EACBAF2AF1L, 0x548F120162451C62L,
+            0x939E303D987B47D7L, 0xD16ED1D631917144L, 0x5F4C95AFC5EDC62EL, 0x1DBC74446C07F0BDL, 0xDAAD56789639AB08L,
+            0x985DB7933FD39D9BL, 0x84193F60D72AF34FL, 0xC6E9DE8B7EC0C5DCL, 0x01F8FCB784FE9E69L, 0x43081D5C2D14A8FAL,
+            0xCD2A5925D9681F90L, 0x8FDAB8CE70822903L, 0x48CB9AF28ABC72B6L, 0x0A3B7B1923564425L, 0x70428B155B4EAF1EL,
+            0x32B26AFEF2A4998DL, 0xF5A348C2089AC238L, 0xB753A929A170F4ABL, 0x3971ED50550C43C1L, 0x7B810CBBFCE67552L,
+            0xBC902E8706D82EE7L, 0xFE60CF6CAF321874L, 0xE224479F47CB76A0L, 0xA0D4A674EE214033L, 0x67C58448141F1B86L,
+            0x253565A3BDF52D15L, 0xAB1721DA49899A7FL, 0xE9E7C031E063ACECL, 0x2EF6E20D1A5DF759L, 0x6C0603E6B3B7C1CAL,
+            0xF6FAE5C07D3274CDL, 0xB40A042BD4D8425EL, 0x731B26172EE619EBL, 0x31EBC7FC870C2F78L, 0xBFC9838573709812L,
+            0xFD39626EDA9AAE81L, 0x3A28405220A4F534L, 0x78D8A1B9894EC3A7L, 0x649C294A61B7AD73L, 0x266CC8A1C85D9BE0L,
+            0xE17DEA9D3263C055L, 0xA38D0B769B89F6C6L, 0x2DAF4F0F6FF541ACL, 0x6F5FAEE4C61F773FL, 0xA84E8CD83C212C8AL,
+            0xEABE6D3395CB1A19L, 0x90C79D3FEDD3F122L, 0xD2377CD44439C7B1L, 0x15265EE8BE079C04L, 0x57D6BF0317EDAA97L,
+            0xD9F4FB7AE3911DFDL, 0x9B041A914A7B2B6EL, 0x5C1538ADB04570DBL, 0x1EE5D94619AF4648L, 0x02A151B5F156289CL,
+            0x4051B05E58BC1E0FL, 0x87409262A28245BAL, 0xC5B073890B687329L, 0x4B9237F0FF14C443L, 0x0962D61B56FEF2D0L,
+            0xCE73F427ACC0A965L, 0x8C8315CC052A9FF6L, 0x3A80143F5CF17F13L, 0x7870F5D4F51B4980L, 0xBF61D7E80F251235L,
+            0xFD913603A6CF24A6L, 0x73B3727A52B393CCL, 0x31439391FB59A55FL, 0xF652B1AD0167FEEAL, 0xB4A25046A88DC879L,
+            0xA8E6D8B54074A6ADL, 0xEA16395EE99E903EL, 0x2D071B6213A0CB8BL, 0x6FF7FA89BA4AFD18L, 0xE1D5BEF04E364A72L,
+            0xA3255F1BE7DC7CE1L, 0x64347D271DE22754L, 0x26C49CCCB40811C7L, 0x5CBD6CC0CC10FAFCL, 0x1E4D8D2B65FACC6FL,
+            0xD95CAF179FC497DAL, 0x9BAC4EFC362EA149L, 0x158E0A85C2521623L, 0x577EEB6E6BB820B0L, 0x906FC95291867B05L,
+            0xD29F28B9386C4D96L, 0xCEDBA04AD0952342L, 0x8C2B41A1797F15D1L, 0x4B3A639D83414E64L, 0x09CA82762AAB78F7L,
+            0x87E8C60FDED7CF9DL, 0xC51827E4773DF90EL, 0x020905D88D03A2BBL, 0x40F9E43324E99428L, 0x2CFFE7D5975E55E2L,
+            0x6E0F063E3EB46371L, 0xA91E2402C48A38C4L, 0xEBEEC5E96D600E57L, 0x65CC8190991CB93DL, 0x273C607B30F68FAEL,
+            0xE02D4247CAC8D41BL, 0xA2DDA3AC6322E288L, 0xBE992B5F8BDB8C5CL, 0xFC69CAB42231BACFL, 0x3B78E888D80FE17AL,
+            0x7988096371E5D7E9L, 0xF7AA4D1A85996083L, 0xB55AACF12C735610L, 0x724B8ECDD64D0DA5L, 0x30BB6F267FA73B36L,
+            0x4AC29F2A07BFD00DL, 0x08327EC1AE55E69EL, 0xCF235CFD546BBD2BL, 0x8DD3BD16FD818BB8L, 0x03F1F96F09FD3CD2L,
+            0x41011884A0170A41L, 0x86103AB85A2951F4L, 0xC4E0DB53F3C36767L, 0xD8A453A01B3A09B3L, 0x9A54B24BB2D03F20L,
+            0x5D45907748EE6495L, 0x1FB5719CE1045206L, 0x919735E51578E56CL, 0xD367D40EBC92D3FFL, 0x1476F63246AC884AL,
+            0x568617D9EF46BED9L, 0xE085162AB69D5E3CL, 0xA275F7C11F7768AFL, 0x6564D5FDE549331AL, 0x279434164CA30589L,
+            0xA9B6706FB8DFB2E3L, 0xEB46918411358470L, 0x2C57B3B8EB0BDFC5L, 0x6EA7525342E1E956L, 0x72E3DAA0AA188782L,
+            0x30133B4B03F2B111L, 0xF7021977F9CCEAA4L, 0xB5F2F89C5026DC37L, 0x3BD0BCE5A45A6B5DL, 0x79205D0E0DB05DCEL,
+            0xBE317F32F78E067BL, 0xFCC19ED95E6430E8L, 0x86B86ED5267CDBD3L, 0xC4488F3E8F96ED40L, 0x0359AD0275A8B6F5L,
+            0x41A94CE9DC428066L, 0xCF8B0890283E370CL, 0x8D7BE97B81D4019FL, 0x4A6ACB477BEA5A2AL, 0x089A2AACD2006CB9L,
+            0x14DEA25F3AF9026DL, 0x562E43B4931334FEL, 0x913F6188692D6F4BL, 0xD3CF8063C0C759D8L, 0x5DEDC41A34BBEEB2L,
+            0x1F1D25F19D51D821L, 0xD80C07CD676F8394L, 0x9AFCE626CE85B507L };
+    private long                crc       = 0;
+
+    @Override
+    public void update(final int b) {
+        update((byte) (b & 0xFF));
+    }
+
+    public void update(final byte b) {
+        final int tab_index = ((int) (this.crc >> 56) ^ b) & 0xFF;
+        this.crc = CRC_TABLE[tab_index] ^ (this.crc << 8);
+    }
+
+    @Override
+    public void update(final byte[] buffer, final int offset, int length) {
+        for (int i = offset; length > 0; length--) {
+            update(buffer[i++]);
+        }
+    }
+
+    public void update(final byte[] buffer) {
+        for (int i = 0; i < buffer.length; i++) {
+            update(buffer[i]);
+        }
+    }
+
+    @Override
+    public long getValue() {
+        return this.crc;
+    }
+
+    @Override
+    public void reset() {
+        this.crc = 0;
+    }
+
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Copiable.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Copiable.java
new file mode 100644
index 0000000..ecef0c8
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Copiable.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Copiable mark interface.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-03 11:15:34 AM 
+ * @param 
+ */
+public interface Copiable {
+
+    /**
+     * Copy current object(deep-clone).
+     */
+    T copy();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CountDownEvent.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CountDownEvent.java
new file mode 100644
index 0000000..0d87156
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CountDownEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * CountDown event.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-17 1:36:36 PM
+ */
+public class CountDownEvent {
+
+    private int             state    = 0;
+    private final Lock      lock     = new ReentrantLock();
+    private final Condition busyCond = this.lock.newCondition();
+    private volatile Object attachment;
+
+    public Object getAttachment() {
+        return this.attachment;
+    }
+
+    public void setAttachment(final Object attachment) {
+        this.attachment = attachment;
+    }
+
+    public int incrementAndGet() {
+        this.lock.lock();
+        try {
+            return ++this.state;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public void countDown() {
+        this.lock.lock();
+        try {
+            if (--this.state == 0) {
+                this.busyCond.signalAll();
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public void await() throws InterruptedException {
+        this.lock.lock();
+        try {
+            while (this.state > 0) {
+                this.busyCond.await();
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CrcUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CrcUtil.java
new file mode 100644
index 0000000..ceef339
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/CrcUtil.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.ByteBuffer;
+
+/**
+ * CRC utilities to compute CRC64 checksum.
+ *
+ * @author boyan(boyan@antfin.com)
+ */
+public final class CrcUtil {
+
+    private static final ThreadLocal CRC_64_THREAD_LOCAL = ThreadLocal.withInitial(CRC64::new);
+
+    /**
+     * Compute CRC64 checksum for byte[].
+     *
+     * @param array source array
+     * @return checksum value
+     */
+    public static long crc64(final byte[] array) {
+        if (array == null) {
+            return 0;
+        }
+        return crc64(array, 0, array.length);
+    }
+
+    /**
+     * Compute CRC64 checksum for byte[].
+     *
+     * @param array  source array
+     * @param offset starting position in the source array
+     * @param length the number of array elements to be computed
+     * @return checksum value
+     */
+    public static long crc64(final byte[] array, final int offset, final int length) {
+        final CRC64 crc64 = CRC_64_THREAD_LOCAL.get();
+        crc64.update(array, offset, length);
+        final long ret = crc64.getValue();
+        crc64.reset();
+        return ret;
+    }
+
+    /**
+     * Compute CRC64 checksum for {@code ByteBuffer}.
+     *
+     * @param buf source {@code ByteBuffer}
+     * @return checksum value
+     */
+    public static long crc64(final ByteBuffer buf) {
+        final int pos = buf.position();
+        final int rem = buf.remaining();
+        if (rem <= 0) {
+            return 0;
+        }
+        // Currently we have not used DirectByteBuffer yet.
+        if (buf.hasArray()) {
+            return crc64(buf.array(), pos + buf.arrayOffset(), rem);
+        }
+        final byte[] b = new byte[rem];
+        buf.mark();
+        buf.get(b);
+        buf.reset();
+        return crc64(b);
+    }
+
+    private CrcUtil() {
+    }
+}
\ No newline at end of file
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DebugStatistics.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DebugStatistics.java
new file mode 100644
index 0000000..b2901a7
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DebugStatistics.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.rocksdb.Statistics;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DebugStatistics extends Statistics {
+
+    public String getString() {
+        return super.toString();
+    }
+
+    @Override
+    public String toString() {
+        // no crash when debug
+        return getClass().getName() + "@" + Integer.toHexString(hashCode());
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Describer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Describer.java
new file mode 100644
index 0000000..da658c8
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Describer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.PrintWriter;
+
+/**
+ * Components that implement this interface need to be able to describe
+ * their own state and output state information via the {@code describe} method.
+ *
+ * @author jiachun.fjc
+ */
+public interface Describer {
+
+    void describe(final Printer out);
+
+    interface Printer {
+
+        /**
+         * Prints an object.
+         *
+         * @param x The Object to be printed
+         * @return this printer
+         */
+        Printer print(final Object x);
+
+        /**
+         * Prints an Object and then terminates the line.
+         *
+         * @param x The Object to be printed.
+         * @return this printer
+         */
+        Printer println(final Object x);
+    }
+
+    class DefaultPrinter implements Describer.Printer {
+
+        private final PrintWriter out;
+
+        public DefaultPrinter(PrintWriter out) {
+            this.out = out;
+        }
+
+        @Override
+        public Describer.Printer print(final Object x) {
+            this.out.print(x);
+            return this;
+        }
+
+        @Override
+        public Describer.Printer println(final Object x) {
+            this.out.println(x);
+            return this;
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DirectExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DirectExecutor.java
new file mode 100644
index 0000000..f4c477a
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DirectExecutor.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.Executor;
+
+/**
+ * An executor that direct run command.
+ * 
+ * @author jiachun.fjc
+ */
+public enum DirectExecutor implements Executor {
+    INSTANCE;
+
+    @Override
+    public void execute(final Runnable command) {
+        command.run();
+    }
+
+    @Override
+    public String toString() {
+        return "DirectExecutor";
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorBuilder.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorBuilder.java
new file mode 100644
index 0000000..f71f58b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorBuilder.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.ThreadFactory;
+
+import com.lmax.disruptor.BlockingWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.WaitStrategy;
+import com.lmax.disruptor.dsl.Disruptor;
+import com.lmax.disruptor.dsl.ProducerType;
+
+/**
+ * A builder to build a disruptor instance.
+ * @author boyan(boyan@antfin.com)
+ *
+ */
+public class DisruptorBuilder {
+    private EventFactory eventFactory;
+    private Integer         ringBufferSize;
+    private ThreadFactory   threadFactory = new NamedThreadFactory("Disruptor-", true);
+    private ProducerType    producerType  = ProducerType.MULTI;
+    private WaitStrategy    waitStrategy  = new BlockingWaitStrategy();
+
+    private DisruptorBuilder() {
+    }
+
+    public static  DisruptorBuilder newInstance() {
+        return new DisruptorBuilder<>();
+    }
+
+    public EventFactory getEventFactory() {
+        return this.eventFactory;
+    }
+
+    public DisruptorBuilder setEventFactory(final EventFactory eventFactory) {
+        this.eventFactory = eventFactory;
+        return this;
+    }
+
+    public int getRingBufferSize() {
+        return this.ringBufferSize;
+    }
+
+    public DisruptorBuilder setRingBufferSize(final int ringBufferSize) {
+        this.ringBufferSize = ringBufferSize;
+        return this;
+    }
+
+    public ThreadFactory getThreadFactory() {
+        return this.threadFactory;
+    }
+
+    public DisruptorBuilder setThreadFactory(final ThreadFactory threadFactory) {
+        this.threadFactory = threadFactory;
+        return this;
+    }
+
+    public ProducerType getProducerType() {
+        return this.producerType;
+    }
+
+    public DisruptorBuilder setProducerType(final ProducerType producerType) {
+        this.producerType = producerType;
+        return this;
+    }
+
+    public WaitStrategy getWaitStrategy() {
+        return this.waitStrategy;
+    }
+
+    public DisruptorBuilder setWaitStrategy(final WaitStrategy waitStrategy) {
+        this.waitStrategy = waitStrategy;
+        return this;
+    }
+
+    public Disruptor build() {
+        Requires.requireNonNull(this.ringBufferSize, " Ring buffer size not set");
+        Requires.requireNonNull(this.eventFactory, "Event factory not set");
+        return new Disruptor<>(this.eventFactory, this.ringBufferSize, this.threadFactory, this.producerType,
+            this.waitStrategy);
+    }
+
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorMetricSet.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorMetricSet.java
new file mode 100644
index 0000000..37d0bc4
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/DisruptorMetricSet.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+import com.lmax.disruptor.RingBuffer;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Disruptor metric set including buffer-size, remaining-capacity etc.
+ */
+public final class DisruptorMetricSet implements MetricSet {
+
+    private final RingBuffer ringBuffer;
+
+    public DisruptorMetricSet(RingBuffer ringBuffer) {
+        super();
+        this.ringBuffer = ringBuffer;
+    }
+
+    /**
+     * Return disruptor metrics
+     * @return disruptor metrics map
+     */
+    @Override
+    public Map getMetrics() {
+        final Map gauges = new HashMap<>();
+        gauges.put("buffer-size", (Gauge) this.ringBuffer::getBufferSize);
+        gauges.put("remaining-capacity", (Gauge) this.ringBuffer::remainingCapacity);
+        return gauges;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Endpoint.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Endpoint.java
new file mode 100644
index 0000000..5921916
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Endpoint.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.Serializable;
+
+/**
+ * A IP address with port.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-12 3:29:12 PM
+ */
+public class Endpoint implements Copiable, Serializable {
+
+    private static final long serialVersionUID = -7329681263115546100L;
+
+    private String            ip               = Utils.IP_ANY;
+    private int               port;
+    private String            str;
+
+    public Endpoint() {
+        super();
+    }
+
+    public Endpoint(String address, int port) {
+        super();
+        this.ip = address;
+        this.port = port;
+    }
+
+    public String getIp() {
+        return this.ip;
+    }
+
+    public int getPort() {
+        return this.port;
+    }
+
+    @Override
+    public String toString() {
+        if (str == null) {
+            str = this.ip + ":" + this.port;
+        }
+        return str;
+    }
+
+    @Override
+    public Endpoint copy() {
+        return new Endpoint(this.ip, this.port);
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + (this.ip == null ? 0 : this.ip.hashCode());
+        result = prime * result + this.port;
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final Endpoint other = (Endpoint) obj;
+        if (this.ip == null) {
+            if (other.ip != null) {
+                return false;
+            }
+        } else if (!this.ip.equals(other.ip)) {
+            return false;
+        }
+        return this.port == other.port;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ExecutorServiceHelper.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ExecutorServiceHelper.java
new file mode 100644
index 0000000..48edae4
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ExecutorServiceHelper.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ExecutorServiceHelper {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ExecutorServiceHelper.class);
+
+    /**
+     * @see #shutdownAndAwaitTermination(ExecutorService, long)
+     */
+    public static boolean shutdownAndAwaitTermination(final ExecutorService pool) {
+        return shutdownAndAwaitTermination(pool, 1000);
+    }
+
+    /**
+     * The following method shuts down an {@code ExecutorService} in two
+     * phases, first by calling {@code shutdown} to reject incoming tasks,
+     * and then calling {@code shutdownNow}, if necessary, to cancel any
+     * lingering tasks.
+     */
+    public static boolean shutdownAndAwaitTermination(final ExecutorService pool, final long timeoutMillis) {
+        if (pool == null) {
+            return true;
+        }
+        // disable new tasks from being submitted
+        pool.shutdown();
+        final TimeUnit unit = TimeUnit.MILLISECONDS;
+        final long phaseOne = timeoutMillis / 5;
+        try {
+            // wait a while for existing tasks to terminate
+            if (pool.awaitTermination(phaseOne, unit)) {
+                return true;
+            }
+            pool.shutdownNow();
+            // wait a while for tasks to respond to being cancelled
+            if (pool.awaitTermination(timeoutMillis - phaseOne, unit)) {
+                return true;
+            }
+            LOG.warn("Fail to shutdown pool: {}.", pool);
+        } catch (final InterruptedException e) {
+            // (Re-)cancel if current thread also interrupted
+            pool.shutdownNow();
+            // preserve interrupt status
+            Thread.currentThread().interrupt();
+        }
+        return false;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/FileOutputSignalHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/FileOutputSignalHandler.java
new file mode 100644
index 0000000..98180a0
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/FileOutputSignalHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.apache.commons.io.FileUtils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class FileOutputSignalHandler implements JRaftSignalHandler {
+
+    protected File getOutputFile(final String path, final String baseFileName) throws IOException {
+        makeDir(path);
+        final String now = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date());
+        final String fileName = baseFileName + "." + now;
+        final File file = Paths.get(path, fileName).toFile();
+        if (!file.exists() && !file.createNewFile()) {
+            throw new IOException("Fail to create file: " + file);
+        }
+        return file;
+    }
+
+    private static void makeDir(final String path) throws IOException {
+        final File dir = Paths.get(path).toFile().getAbsoluteFile();
+        if (dir.exists()) {
+            Requires.requireTrue(dir.isDirectory(), String.format("[%s] is not directory.", path));
+        } else {
+            FileUtils.forceMkdir(dir);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/HeapByteBufUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/HeapByteBufUtil.java
new file mode 100644
index 0000000..e9ed09b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/HeapByteBufUtil.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2015 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License,
+ * version 2.0 (the "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at:
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Utility class for heap buffers.
+ *
+ * Forked from Netty.
+ */
+final class HeapByteBufUtil {
+
+    static byte getByte(byte[] memory, int index) {
+        return memory[index];
+    }
+
+    static short getShort(byte[] memory, int index) {
+        return (short) (memory[index] << 8 | memory[index + 1] & 0xFF);
+    }
+
+    static short getShortLE(byte[] memory, int index) {
+        return (short) (memory[index] & 0xff | memory[index + 1] << 8);
+    }
+
+    static int getUnsignedMedium(byte[] memory, int index) {
+        return (memory[index] & 0xff) << 16 | (memory[index + 1] & 0xff) << 8 | memory[index + 2] & 0xff;
+    }
+
+    static int getUnsignedMediumLE(byte[] memory, int index) {
+        return memory[index] & 0xff | (memory[index + 1] & 0xff) << 8 | (memory[index + 2] & 0xff) << 16;
+    }
+
+    static int getInt(byte[] memory, int index) {
+        return (memory[index] & 0xff) << 24 | (memory[index + 1] & 0xff) << 16 | (memory[index + 2] & 0xff) << 8
+               | memory[index + 3] & 0xff;
+    }
+
+    static int getIntLE(byte[] memory, int index) {
+        return memory[index] & 0xff | (memory[index + 1] & 0xff) << 8 | (memory[index + 2] & 0xff) << 16
+               | (memory[index + 3] & 0xff) << 24;
+    }
+
+    static long getLong(byte[] memory, int index) {
+        return ((long) memory[index] & 0xff) << 56 | ((long) memory[index + 1] & 0xff) << 48
+               | ((long) memory[index + 2] & 0xff) << 40 | ((long) memory[index + 3] & 0xff) << 32
+               | ((long) memory[index + 4] & 0xff) << 24 | ((long) memory[index + 5] & 0xff) << 16
+               | ((long) memory[index + 6] & 0xff) << 8 | (long) memory[index + 7] & 0xff;
+    }
+
+    static long getLongLE(byte[] memory, int index) {
+        return (long) memory[index] & 0xff | ((long) memory[index + 1] & 0xff) << 8
+               | ((long) memory[index + 2] & 0xff) << 16 | ((long) memory[index + 3] & 0xff) << 24
+               | ((long) memory[index + 4] & 0xff) << 32 | ((long) memory[index + 5] & 0xff) << 40
+               | ((long) memory[index + 6] & 0xff) << 48 | ((long) memory[index + 7] & 0xff) << 56;
+    }
+
+    static void setByte(byte[] memory, int index, int value) {
+        memory[index] = (byte) value;
+    }
+
+    static void setShort(byte[] memory, int index, int value) {
+        memory[index] = (byte) (value >>> 8);
+        memory[index + 1] = (byte) value;
+    }
+
+    static void setShortLE(byte[] memory, int index, int value) {
+        memory[index] = (byte) value;
+        memory[index + 1] = (byte) (value >>> 8);
+    }
+
+    static void setMedium(byte[] memory, int index, int value) {
+        memory[index] = (byte) (value >>> 16);
+        memory[index + 1] = (byte) (value >>> 8);
+        memory[index + 2] = (byte) value;
+    }
+
+    static void setMediumLE(byte[] memory, int index, int value) {
+        memory[index] = (byte) value;
+        memory[index + 1] = (byte) (value >>> 8);
+        memory[index + 2] = (byte) (value >>> 16);
+    }
+
+    static void setInt(byte[] memory, int index, int value) {
+        memory[index] = (byte) (value >>> 24);
+        memory[index + 1] = (byte) (value >>> 16);
+        memory[index + 2] = (byte) (value >>> 8);
+        memory[index + 3] = (byte) value;
+    }
+
+    static void setIntLE(byte[] memory, int index, int value) {
+        memory[index] = (byte) value;
+        memory[index + 1] = (byte) (value >>> 8);
+        memory[index + 2] = (byte) (value >>> 16);
+        memory[index + 3] = (byte) (value >>> 24);
+    }
+
+    static void setLong(byte[] memory, int index, long value) {
+        memory[index] = (byte) (value >>> 56);
+        memory[index + 1] = (byte) (value >>> 48);
+        memory[index + 2] = (byte) (value >>> 40);
+        memory[index + 3] = (byte) (value >>> 32);
+        memory[index + 4] = (byte) (value >>> 24);
+        memory[index + 5] = (byte) (value >>> 16);
+        memory[index + 6] = (byte) (value >>> 8);
+        memory[index + 7] = (byte) value;
+    }
+
+    static void setLongLE(byte[] memory, int index, long value) {
+        memory[index] = (byte) value;
+        memory[index + 1] = (byte) (value >>> 8);
+        memory[index + 2] = (byte) (value >>> 16);
+        memory[index + 3] = (byte) (value >>> 24);
+        memory[index + 4] = (byte) (value >>> 32);
+        memory[index + 5] = (byte) (value >>> 40);
+        memory[index + 6] = (byte) (value >>> 48);
+        memory[index + 7] = (byte) (value >>> 56);
+    }
+
+    private HeapByteBufUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Ints.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Ints.java
new file mode 100644
index 0000000..375e572
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Ints.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Static utility methods pertaining to {@code int} primitives.
+ *
+ * @author jiachun.fjc
+ */
+public final class Ints {
+
+    /**
+     * The largest power of two that can be represented as an int.
+     */
+    public static final int MAX_POWER_OF_TWO = 1 << (Integer.SIZE - 2);
+
+    /**
+     * Returns the {@code int} value that is equal to {@code value}, if possible.
+     */
+    public static int checkedCast(final long value) {
+        final int result = (int) value;
+        Requires.requireTrue(result == value, "out of range: " + value);
+        return result;
+    }
+
+    /**
+     * Returns the {@code int} nearest in value to {@code value}.
+     */
+    public static int saturatedCast(final long value) {
+        return value > Integer.MAX_VALUE ? Integer.MAX_VALUE : value < Integer.MIN_VALUE ? Integer.MIN_VALUE
+            : (int) value;
+    }
+
+    /**
+     * Fast method of finding the next power of 2 greater than or equal to the supplied value.
+     *
+     * If the value is {@code <= 0} then 1 will be returned.
+     * This method is not suitable for {@link Integer#MIN_VALUE} or numbers greater than 2^30.
+     *
+     * @param value from which to search for next power of 2
+     * @return The next power of 2 or the value itself if it is a power of 2
+     */
+    public static int findNextPositivePowerOfTwo(final int value) {
+        return value <= 0 ? 1 : value >= 0x40000000 ? 0x40000000 : 1 << (32 - Integer.numberOfLeadingZeros(value - 1));
+    }
+
+    /**
+     * Find the next larger positive power of two value up from the given value.
+     * If value is a power of two then this value will be returned.
+     *
+     * @param value from which next positive power of two will be found
+     * @return the next positive power of 2 or this value if it is a power of 2
+     */
+    public static int roundToPowerOfTwo(final int value) {
+        return 1 << (32 - Integer.numberOfLeadingZeros(value - 1));
+    }
+
+    /**
+     * Is this value a power of two.
+     *
+     * @param value to be tested to see if it is a power of two
+     * @return true if the value is a power of 2 otherwise false
+     */
+    public static boolean isPowerOfTwo(final int value) {
+        return (value & (value - 1)) == 0;
+    }
+
+    private Ints() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftServiceLoader.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftServiceLoader.java
new file mode 100644
index 0000000..1073b35
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftServiceLoader.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.ServiceConfigurationError;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A simple service-provider loading facility (SPI).
+ *
+ * @author jiachun.fjc
+ */
+public final class JRaftServiceLoader implements Iterable {
+
+    private static final Logger      LOG       = LoggerFactory.getLogger(JRaftServiceLoader.class);
+
+    private static final String      PREFIX    = "META-INF/services/";
+
+    // the class or interface representing the service being loaded
+    private final Class           service;
+
+    // the class loader used to locate, load, and instantiate providers
+    private final ClassLoader        loader;
+
+    // cached providers, in instantiation order
+    private LinkedHashMap providers = new LinkedHashMap<>();
+
+    // the current lazy-lookup iterator
+    private LazyIterator             lookupIterator;
+
+    public static  JRaftServiceLoader load(final Class service) {
+        return JRaftServiceLoader.load(service, Thread.currentThread().getContextClassLoader());
+    }
+
+    public static  JRaftServiceLoader load(final Class service, final ClassLoader loader) {
+        return new JRaftServiceLoader<>(service, loader);
+    }
+
+    public List sort() {
+        final Iterator it = iterator();
+        final List sortList = new ArrayList<>();
+        while (it.hasNext()) {
+            sortList.add(it.next());
+        }
+
+        if (sortList.size() <= 1) {
+            return sortList;
+        }
+
+        sortList.sort((o1, o2) -> {
+            final SPI o1Spi = o1.getClass().getAnnotation(SPI.class);
+            final SPI o2Spi = o2.getClass().getAnnotation(SPI.class);
+
+            final int o1Priority = o1Spi == null ? 0 : o1Spi.priority();
+            final int o2Priority = o2Spi == null ? 0 : o2Spi.priority();
+
+            return -(o1Priority - o2Priority);
+        });
+
+        return sortList;
+    }
+
+    public S first() {
+        final Iterator> it = classIterator();
+        Class first = null;
+        while (it.hasNext()) {
+            final Class cls = it.next();
+            if (first == null) {
+                first = cls;
+            } else {
+                final SPI currSpi = first.getAnnotation(SPI.class);
+                final SPI nextSpi = cls.getAnnotation(SPI.class);
+
+                final int currPriority = currSpi == null ? 0 : currSpi.priority();
+                final int nextPriority = nextSpi == null ? 0 : nextSpi.priority();
+
+                if (nextPriority > currPriority) {
+                    first = cls;
+                }
+            }
+        }
+
+        if (first == null) {
+            throw fail(this.service, "could not find any implementation for class");
+        }
+
+        final S ins = this.providers.get(first.getName());
+        if (ins != null) {
+            return ins;
+        }
+
+        return newProvider(first);
+    }
+
+    public S find(final String implName) {
+        for (final S s : this.providers.values()) {
+            final SPI spi = s.getClass().getAnnotation(SPI.class);
+            if (spi != null && spi.name().equalsIgnoreCase(implName)) {
+                return s;
+            }
+        }
+        while (this.lookupIterator.hasNext()) {
+            final Class cls = this.lookupIterator.next();
+            final SPI spi = cls.getAnnotation(SPI.class);
+            if (spi != null && spi.name().equalsIgnoreCase(implName)) {
+                try {
+                    return newProvider(cls);
+                } catch (final Throwable x) {
+                    throw fail(this.service, "provider " + cls.getName() + " could not be instantiated", x);
+                }
+            }
+        }
+        throw fail(this.service, "provider " + implName + " not found");
+    }
+
+    public void reload() {
+        this.providers.clear();
+        this.lookupIterator = new LazyIterator(this.service, this.loader);
+    }
+
+    private JRaftServiceLoader(final Class service, final ClassLoader loader) {
+        this.service = Requires.requireNonNull(service, "service interface cannot be null");
+        this.loader = (loader == null) ? ClassLoader.getSystemClassLoader() : loader;
+        reload();
+    }
+
+    private static ServiceConfigurationError fail(final Class service, final String msg, final Throwable cause) {
+        return new ServiceConfigurationError(service.getName() + ": " + msg, cause);
+    }
+
+    private static ServiceConfigurationError fail(final Class service, final String msg) {
+        return new ServiceConfigurationError(service.getName() + ": " + msg);
+    }
+
+    private static ServiceConfigurationError fail(final Class service, final URL url, final int line,
+                                                  final String msg) {
+        return fail(service, url + ":" + line + ": " + msg);
+    }
+
+    // parse a single line from the given configuration file, adding the name
+    // on the line to the names list.
+    private int parseLine(final Class service, final URL u, final BufferedReader r, final int lc,
+                          final List names) throws IOException, ServiceConfigurationError {
+
+        String ln = r.readLine();
+        if (ln == null) {
+            return -1;
+        }
+        final int ci = ln.indexOf('#');
+        if (ci >= 0) {
+            ln = ln.substring(0, ci);
+        }
+        ln = ln.trim();
+        final int n = ln.length();
+        if (n != 0) {
+            if ((ln.indexOf(' ') >= 0) || (ln.indexOf('\t') >= 0)) {
+                throw fail(service, u, lc, "illegal configuration-file syntax");
+            }
+            int cp = ln.codePointAt(0);
+            if (!Character.isJavaIdentifierStart(cp)) {
+                throw fail(service, u, lc, "illegal provider-class name: " + ln);
+            }
+            for (int i = Character.charCount(cp); i < n; i += Character.charCount(cp)) {
+                cp = ln.codePointAt(i);
+                if (!Character.isJavaIdentifierPart(cp) && (cp != '.')) {
+                    throw fail(service, u, lc, "Illegal provider-class name: " + ln);
+                }
+            }
+            if (!this.providers.containsKey(ln) && !names.contains(ln)) {
+                names.add(ln);
+            }
+        }
+        return lc + 1;
+    }
+
+    private Iterator parse(final Class service, final URL url) {
+        final ArrayList names = new ArrayList<>();
+        try (final InputStream in = url.openStream();
+                final BufferedReader r = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
+            int lc = 1;
+            // noinspection StatementWithEmptyBody
+            while ((lc = parseLine(service, url, r, lc, names)) >= 0)
+                ;
+        } catch (final IOException x) {
+            throw fail(service, "error reading configuration file", x);
+        }
+        return names.iterator();
+    }
+
+    @Override
+    public Iterator iterator() {
+        return new Iterator() {
+
+            final Iterator> knownProviders = JRaftServiceLoader.this.providers.entrySet()
+                                                                    .iterator();
+
+            @Override
+            public boolean hasNext() {
+                return this.knownProviders.hasNext() || JRaftServiceLoader.this.lookupIterator.hasNext();
+            }
+
+            @Override
+            public S next() {
+                if (this.knownProviders.hasNext()) {
+                    return this.knownProviders.next().getValue();
+                }
+                final Class cls = JRaftServiceLoader.this.lookupIterator.next();
+                return newProvider(cls);
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    public Iterator> classIterator() {
+        return new Iterator>() {
+
+            final Iterator> knownProviders = JRaftServiceLoader.this.providers.entrySet()
+                                                                    .iterator();
+
+            @Override
+            public boolean hasNext() {
+                return this.knownProviders.hasNext() || JRaftServiceLoader.this.lookupIterator.hasNext();
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public Class next() {
+                if (this.knownProviders.hasNext()) {
+                    return (Class) this.knownProviders.next().getValue().getClass();
+                }
+                return JRaftServiceLoader.this.lookupIterator.next();
+            }
+
+            @Override
+            public void remove() {
+                throw new UnsupportedOperationException();
+            }
+        };
+    }
+
+    private S newProvider(final Class cls) {
+        LOG.info("SPI service [{} - {}] loading.", this.service.getName(), cls.getName());
+        try {
+            final S provider = this.service.cast(cls.newInstance());
+            this.providers.put(cls.getName(), provider);
+            return provider;
+        } catch (final Throwable x) {
+            throw fail(this.service, "provider " + cls.getName() + " could not be instantiated", x);
+        }
+    }
+
+    private class LazyIterator implements Iterator> {
+        Class         service;
+        ClassLoader      loader;
+        Enumeration configs  = null;
+        Iterator pending  = null;
+        String           nextName = null;
+
+        private LazyIterator(Class service, ClassLoader loader) {
+            this.service = service;
+            this.loader = loader;
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (this.nextName != null) {
+                return true;
+            }
+            if (this.configs == null) {
+                try {
+                    final String fullName = PREFIX + this.service.getName();
+                    if (this.loader == null) {
+                        this.configs = ClassLoader.getSystemResources(fullName);
+                    } else {
+                        this.configs = this.loader.getResources(fullName);
+                    }
+                } catch (final IOException x) {
+                    throw fail(this.service, "error locating configuration files", x);
+                }
+            }
+            while ((this.pending == null) || !this.pending.hasNext()) {
+                if (!this.configs.hasMoreElements()) {
+                    return false;
+                }
+                this.pending = parse(this.service, this.configs.nextElement());
+            }
+            this.nextName = this.pending.next();
+            return true;
+        }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public Class next() {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            final String name = this.nextName;
+            this.nextName = null;
+            final Class cls;
+            try {
+                cls = Class.forName(name, false, this.loader);
+            } catch (final ClassNotFoundException x) {
+                throw fail(this.service, "provider " + name + " not found");
+            }
+            if (!this.service.isAssignableFrom(cls)) {
+                throw fail(this.service, "provider " + name + " not a subtype");
+            }
+            return (Class) cls;
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    /**
+     * Returns a string describing this service.
+     */
+    @Override
+    public String toString() {
+        return "com.alipay.sofa.jraft.util.JRaftServiceLoader[" + this.service.getName() + "]";
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftSignalHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftSignalHandler.java
new file mode 100644
index 0000000..fa46cef
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/JRaftSignalHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface JRaftSignalHandler {
+
+    void handle(final String signalName);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogExceptionHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogExceptionHandler.java
new file mode 100644
index 0000000..d70c64b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogExceptionHandler.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.lmax.disruptor.ExceptionHandler;
+
+/**
+ * Disruptor exception handler.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-05 9:31:28 PM
+ */
+public final class LogExceptionHandler implements ExceptionHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LogExceptionHandler.class);
+
+    public interface OnEventException {
+
+        void onException(T event, Throwable ex);
+    }
+
+    private final String              name;
+    private final OnEventException onEventException;
+
+    public LogExceptionHandler(String name) {
+        this(name, null);
+    }
+
+    public LogExceptionHandler(String name, OnEventException onEventException) {
+        this.name = name;
+        this.onEventException = onEventException;
+    }
+
+    @Override
+    public void handleOnStartException(Throwable ex) {
+        LOG.error("Fail to start {} disruptor", this.name, ex);
+    }
+
+    @Override
+    public void handleOnShutdownException(Throwable ex) {
+        LOG.error("Fail to shutdown {} disruptor", this.name, ex);
+    }
+
+    @Override
+    public void handleEventException(Throwable ex, long sequence, T event) {
+        LOG.error("Handle {} disruptor event error, event is {}", this.name, event, ex);
+        if (this.onEventException != null) {
+            this.onEventException.onException(event, ex);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogScheduledThreadPoolExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogScheduledThreadPoolExecutor.java
new file mode 100644
index 0000000..4f6b5f1
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogScheduledThreadPoolExecutor.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link java.util.concurrent.ThreadPoolExecutor} that can additionally
+ * schedule commands to run after a given delay with a logger witch can print
+ * error message for failed execution.
+ *
+ * @author jiachun.fjc
+ */
+public class LogScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LogScheduledThreadPoolExecutor.class);
+
+    private final String        name;
+
+    public LogScheduledThreadPoolExecutor(int corePoolSize, String name) {
+        super(corePoolSize);
+        this.name = name;
+    }
+
+    public LogScheduledThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory, String name) {
+        super(corePoolSize, threadFactory);
+        this.name = name;
+    }
+
+    public LogScheduledThreadPoolExecutor(int corePoolSize, RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, handler);
+        this.name = name;
+    }
+
+    public LogScheduledThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory,
+                                          RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, threadFactory, handler);
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+        if (t == null && r instanceof Future) {
+            try {
+                final Future f = (Future) r;
+                if (f.isDone()) {
+                    f.get();
+                }
+            } catch (final CancellationException ce) {
+                // ignored
+            } catch (final ExecutionException ee) {
+                t = ee.getCause();
+            } catch (final InterruptedException ie) {
+                Thread.currentThread().interrupt(); // ignore/reset
+            }
+        }
+        if (t != null) {
+            LOG.error("Uncaught exception in pool: {}, {}.", this.name, super.toString(), t);
+        }
+    }
+
+    @Override
+    protected void terminated() {
+        super.terminated();
+        LOG.info("ThreadPool is terminated: {}, {}.", this.name, super.toString());
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogThreadPoolExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogThreadPoolExecutor.java
new file mode 100644
index 0000000..4b0164e
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/LogThreadPoolExecutor.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A {@link java.util.concurrent.ExecutorService} that witch can print
+ * error message for failed execution.
+ *
+ * @author jiachun.fjc
+ */
+public class LogThreadPoolExecutor extends ThreadPoolExecutor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LogThreadPoolExecutor.class);
+
+    private final String        name;
+
+    public LogThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                 BlockingQueue workQueue, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
+        this.name = name;
+    }
+
+    public LogThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                 BlockingQueue workQueue, ThreadFactory threadFactory, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory);
+        this.name = name;
+    }
+
+    public LogThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                 BlockingQueue workQueue, RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler);
+        this.name = name;
+    }
+
+    public LogThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                 BlockingQueue workQueue, ThreadFactory threadFactory,
+                                 RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler);
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    @Override
+    protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+        if (t == null && r instanceof Future) {
+            try {
+                final Future f = (Future) r;
+                if (f.isDone()) {
+                    f.get();
+                }
+            } catch (final CancellationException ce) {
+                // ignored
+            } catch (final ExecutionException ee) {
+                t = ee.getCause();
+            } catch (final InterruptedException ie) {
+                Thread.currentThread().interrupt(); // ignore/reset
+            }
+        }
+        if (t != null) {
+            LOG.error("Uncaught exception in pool: {}, {}.", this.name, super.toString(), t);
+        }
+    }
+
+    @Override
+    protected void terminated() {
+        super.terminated();
+        LOG.info("ThreadPool is terminated: {}, {}.", this.name, super.toString());
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricReporter.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricReporter.java
new file mode 100644
index 0000000..39a24ad
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricReporter.java
@@ -0,0 +1,436 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.PrintStream;
+import java.text.DateFormat;
+import java.util.Collections;
+import java.util.Date;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Clock;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricAttribute;
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Snapshot;
+import com.codahale.metrics.Timer;
+
+/**
+ * A reporter which outputs measurements to a {@link PrintStream}, like {@code System.out}.
+ *
+ * Fork form {@link com.codahale.metrics.ConsoleReporter}
+ */
+public class MetricReporter {
+
+    /**
+     * Returns a new {@link Builder} for {@link MetricReporter}.
+     *
+     * @param registry the registry to report
+     * @return a {@link Builder} instance for a {@link MetricReporter}
+     */
+    public static Builder forRegistry(final MetricRegistry registry) {
+        return new Builder(registry);
+    }
+
+    /**
+     * Report the current values of all metrics in the registry.
+     */
+    public void report() {
+        synchronized (this) {
+            report(this.registry.getGauges(this.filter), //
+                this.registry.getCounters(this.filter), //
+                this.registry.getHistograms(this.filter), //
+                this.registry.getMeters(this.filter), //
+                this.registry.getTimers(this.filter));
+        }
+    }
+
+    /**
+     * A builder for {@link MetricReporter} instances. Defaults to using the default locale and
+     * time zone, writing to {@code System.out}, converting rates to events/second, converting
+     * durations to milliseconds, and not filtering metrics.
+     */
+    public static class Builder {
+
+        private final MetricRegistry registry;
+
+        private String               prefix;
+        private PrintStream          output;
+        private Locale               locale;
+        private Clock                clock;
+        private TimeZone             timeZone;
+        private TimeUnit             rateUnit;
+        private TimeUnit             durationUnit;
+        private MetricFilter         filter;
+        private Set disabledMetricAttributes;
+
+        private Builder(MetricRegistry registry) {
+            this.registry = registry;
+            this.prefix = "";
+            this.output = System.out;
+            this.locale = Locale.getDefault();
+            this.clock = Clock.defaultClock();
+            this.timeZone = TimeZone.getDefault();
+            this.rateUnit = TimeUnit.SECONDS;
+            this.durationUnit = TimeUnit.MILLISECONDS;
+            this.filter = MetricFilter.ALL;
+            this.disabledMetricAttributes = Collections.emptySet();
+        }
+
+        /**
+         * Prefix all metric names with the given string.
+         *
+         * @param prefix the prefix for all banner names
+         * @return {@code this}
+         */
+        public Builder prefixedWith(final String prefix) {
+            this.prefix = prefix;
+            return this;
+        }
+
+        /**
+         * Write to the given {@link PrintStream}.
+         *
+         * @param output a {@link PrintStream} instance.
+         * @return {@code this}
+         */
+        public Builder outputTo(final PrintStream output) {
+            this.output = output;
+            return this;
+        }
+
+        /**
+         * Format numbers for the given {@link Locale}.
+         *
+         * @param locale a {@link Locale}
+         * @return {@code this}
+         */
+        public Builder formattedFor(final Locale locale) {
+            this.locale = locale;
+            return this;
+        }
+
+        /**
+         * Use the given {@link Clock} instance for the time.
+         *
+         * @param clock a {@link Clock} instance
+         * @return {@code this}
+         */
+        public Builder withClock(final Clock clock) {
+            this.clock = clock;
+            return this;
+        }
+
+        /**
+         * Use the given {@link TimeZone} for the time.
+         *
+         * @param timeZone a {@link TimeZone}
+         * @return {@code this}
+         */
+        public Builder formattedFor(final TimeZone timeZone) {
+            this.timeZone = timeZone;
+            return this;
+        }
+
+        /**
+         * Convert rates to the given time unit.
+         *
+         * @param rateUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertRatesTo(final TimeUnit rateUnit) {
+            this.rateUnit = rateUnit;
+            return this;
+        }
+
+        /**
+         * Convert durations to the given time unit.
+         *
+         * @param durationUnit a unit of time
+         * @return {@code this}
+         */
+        public Builder convertDurationsTo(final TimeUnit durationUnit) {
+            this.durationUnit = durationUnit;
+            return this;
+        }
+
+        /**
+         * Only report metrics which match the given filter.
+         *
+         * @param filter a {@link MetricFilter}
+         * @return {@code this}
+         */
+        public Builder filter(final MetricFilter filter) {
+            this.filter = filter;
+            return this;
+        }
+
+        /**
+         * Don't report the passed metric attributes for all metrics (e.g. "p999", "stddev" or "m15").
+         * See {@link MetricAttribute}.
+         *
+         * @param disabledMetricAttributes a {@link MetricFilter}
+         * @return {@code this}
+         */
+        public Builder disabledMetricAttributes(final Set disabledMetricAttributes) {
+            this.disabledMetricAttributes = disabledMetricAttributes;
+            return this;
+        }
+
+        /**
+         * Builds a {@link MetricReporter} with the given properties.
+         *
+         * @return a {@link MetricReporter}
+         */
+        public MetricReporter build() {
+            return new MetricReporter(this.registry, //
+                this.output, //
+                this.prefix, //
+                this.locale, //
+                this.clock, //
+                this.timeZone, //
+                this.rateUnit, //
+                this.durationUnit, //
+                this.filter, //
+                this.disabledMetricAttributes);
+        }
+    }
+
+    private static final int           CONSOLE_WIDTH = 80;
+
+    private final MetricRegistry       registry;
+    private final Set disabledMetricAttributes;
+    private final MetricFilter         filter;
+    private final long                 durationFactor;
+    private final String               durationUnit;
+    private final long                 rateFactor;
+    private final String               rateUnit;
+    private final String               prefix;
+    private final PrintStream          output;
+    private final Locale               locale;
+    private final Clock                clock;
+    private final DateFormat           dateFormat;
+
+    private MetricReporter(MetricRegistry registry, //
+                           PrintStream output, //
+                           String prefix, //
+                           Locale locale, //
+                           Clock clock, //
+                           TimeZone timeZone, //
+                           TimeUnit rateUnit, //
+                           TimeUnit durationUnit, //
+                           MetricFilter filter, //
+                           Set disabledMetricAttributes) {
+        this.registry = registry;
+        this.output = output;
+        this.prefix = prefix;
+        this.locale = locale;
+        this.clock = clock;
+        this.dateFormat = DateFormat.getDateTimeInstance(DateFormat.SHORT, DateFormat.MEDIUM, locale);
+        this.dateFormat.setTimeZone(timeZone);
+        this.rateFactor = rateUnit.toSeconds(1);
+        this.rateUnit = calculateRateUnit(rateUnit);
+        this.durationFactor = durationUnit.toNanos(1);
+        this.durationUnit = durationUnit.toString().toLowerCase(Locale.US);
+        this.filter = filter;
+        this.disabledMetricAttributes = disabledMetricAttributes != null ? disabledMetricAttributes : Collections
+            .emptySet();
+    }
+
+    public void report(final SortedMap gauges, final SortedMap counters,
+                       final SortedMap histograms, final SortedMap meters,
+                       final SortedMap timers) {
+        final String dateTime = this.dateFormat.format(new Date(this.clock.getTime()));
+        printWithBanner(dateTime, '=');
+        this.output.println();
+
+        if (!gauges.isEmpty()) {
+            printWithBanner("-- Gauges", '-');
+            for (final Map.Entry entry : gauges.entrySet()) {
+                this.output.println(entry.getKey());
+                printGauge(entry.getValue());
+            }
+            this.output.println();
+        }
+
+        if (!counters.isEmpty()) {
+            printWithBanner("-- Counters", '-');
+            for (final Map.Entry entry : counters.entrySet()) {
+                this.output.println(entry.getKey());
+                printCounter(entry);
+            }
+            this.output.println();
+        }
+
+        if (!histograms.isEmpty()) {
+            printWithBanner("-- Histograms", '-');
+            for (final Map.Entry entry : histograms.entrySet()) {
+                this.output.println(entry.getKey());
+                printHistogram(entry.getValue());
+            }
+            this.output.println();
+        }
+
+        if (!meters.isEmpty()) {
+            printWithBanner("-- Meters", '-');
+            for (final Map.Entry entry : meters.entrySet()) {
+                this.output.println(entry.getKey());
+                printMeter(entry.getValue());
+            }
+            this.output.println();
+        }
+
+        if (!timers.isEmpty()) {
+            printWithBanner("-- Timers", '-');
+            for (Map.Entry entry : timers.entrySet()) {
+                this.output.println(entry.getKey());
+                printTimer(entry.getValue());
+            }
+            this.output.println();
+        }
+
+        this.output.println();
+        this.output.flush();
+    }
+
+    private void printMeter(final Meter meter) {
+        printIfEnabled(MetricAttribute.COUNT, String.format(this.locale, "             count = %d", meter.getCount()));
+        printIfEnabled(MetricAttribute.MEAN_RATE, String.format(this.locale, "         mean rate = %2.2f events/%s",
+            convertRate(meter.getMeanRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M1_RATE, String.format(this.locale, "     1-minute rate = %2.2f events/%s",
+            convertRate(meter.getOneMinuteRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M5_RATE, String.format(this.locale, "     5-minute rate = %2.2f events/%s",
+            convertRate(meter.getFiveMinuteRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M15_RATE, String.format(this.locale, "    15-minute rate = %2.2f events/%s",
+            convertRate(meter.getFifteenMinuteRate()), this.rateUnit));
+    }
+
+    private void printCounter(final Map.Entry entry) {
+        this.output.printf(this.locale, "             count = %d%n", entry.getValue().getCount());
+    }
+
+    private void printGauge(final Gauge gauge) {
+        this.output.printf(this.locale, "             value = %s%n", gauge.getValue());
+    }
+
+    private void printHistogram(final Histogram histogram) {
+        printIfEnabled(MetricAttribute.COUNT,
+            String.format(this.locale, "             count = %d", histogram.getCount()));
+        final Snapshot snapshot = histogram.getSnapshot();
+        printIfEnabled(MetricAttribute.MIN, String.format(this.locale, "               min = %d", snapshot.getMin()));
+        printIfEnabled(MetricAttribute.MAX, String.format(this.locale, "               max = %d", snapshot.getMax()));
+        printIfEnabled(MetricAttribute.MEAN,
+            String.format(this.locale, "              mean = %2.2f", snapshot.getMean()));
+        printIfEnabled(MetricAttribute.STDDEV,
+            String.format(this.locale, "            stddev = %2.2f", snapshot.getStdDev()));
+        printIfEnabled(MetricAttribute.P50,
+            String.format(this.locale, "            median = %2.2f", snapshot.getMedian()));
+        printIfEnabled(MetricAttribute.P75,
+            String.format(this.locale, "              75%% <= %2.2f", snapshot.get75thPercentile()));
+        printIfEnabled(MetricAttribute.P95,
+            String.format(this.locale, "              95%% <= %2.2f", snapshot.get95thPercentile()));
+        printIfEnabled(MetricAttribute.P98,
+            String.format(this.locale, "              98%% <= %2.2f", snapshot.get98thPercentile()));
+        printIfEnabled(MetricAttribute.P99,
+            String.format(this.locale, "              99%% <= %2.2f", snapshot.get99thPercentile()));
+        printIfEnabled(MetricAttribute.P999,
+            String.format(this.locale, "            99.9%% <= %2.2f", snapshot.get999thPercentile()));
+    }
+
+    private void printTimer(final Timer timer) {
+        final Snapshot snapshot = timer.getSnapshot();
+        printIfEnabled(MetricAttribute.COUNT, String.format(this.locale, "             count = %d", timer.getCount()));
+        printIfEnabled(MetricAttribute.MEAN_RATE, String.format(this.locale, "         mean rate = %2.2f calls/%s",
+            convertRate(timer.getMeanRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M1_RATE, String.format(this.locale, "     1-minute rate = %2.2f calls/%s",
+            convertRate(timer.getOneMinuteRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M5_RATE, String.format(this.locale, "     5-minute rate = %2.2f calls/%s",
+            convertRate(timer.getFiveMinuteRate()), this.rateUnit));
+        printIfEnabled(MetricAttribute.M15_RATE, String.format(this.locale, "    15-minute rate = %2.2f calls/%s",
+            convertRate(timer.getFifteenMinuteRate()), this.rateUnit));
+
+        printIfEnabled(MetricAttribute.MIN, String.format(this.locale, "               min = %2.2f %s",
+            convertDuration(snapshot.getMin()), this.durationUnit));
+        printIfEnabled(MetricAttribute.MAX, String.format(this.locale, "               max = %2.2f %s",
+            convertDuration(snapshot.getMax()), this.durationUnit));
+        printIfEnabled(MetricAttribute.MEAN, String.format(this.locale, "              mean = %2.2f %s",
+            convertDuration(snapshot.getMean()), this.durationUnit));
+        printIfEnabled(MetricAttribute.STDDEV, String.format(this.locale, "            stddev = %2.2f %s",
+            convertDuration(snapshot.getStdDev()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P50, String.format(this.locale, "            median = %2.2f %s",
+            convertDuration(snapshot.getMedian()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P75, String.format(this.locale, "              75%% <= %2.2f %s",
+            convertDuration(snapshot.get75thPercentile()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P95, String.format(this.locale, "              95%% <= %2.2f %s",
+            convertDuration(snapshot.get95thPercentile()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P98, String.format(this.locale, "              98%% <= %2.2f %s",
+            convertDuration(snapshot.get98thPercentile()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P99, String.format(this.locale, "              99%% <= %2.2f %s",
+            convertDuration(snapshot.get99thPercentile()), this.durationUnit));
+        printIfEnabled(MetricAttribute.P999, String.format(this.locale, "            99.9%% <= %2.2f %s",
+            convertDuration(snapshot.get999thPercentile()), this.durationUnit));
+    }
+
+    private void printWithBanner(final String s, final char c) {
+        if (!this.prefix.isEmpty()) {
+            this.output.print(this.prefix);
+            this.output.print(' ');
+        }
+        this.output.print(s);
+        this.output.print(' ');
+        for (int i = 0; i < (CONSOLE_WIDTH - s.length() - 1); i++) {
+            this.output.print(c);
+        }
+        this.output.println();
+    }
+
+    /**
+     * Print only if the attribute is enabled
+     *
+     * @param type   Metric attribute
+     * @param status Status to be logged
+     */
+    private void printIfEnabled(final MetricAttribute type, final String status) {
+        if (this.disabledMetricAttributes.contains(type)) {
+            return;
+        }
+
+        this.output.println(status);
+    }
+
+    private String calculateRateUnit(final TimeUnit unit) {
+        final String s = unit.toString().toLowerCase(Locale.US);
+        return s.substring(0, s.length() - 1);
+    }
+
+    private double convertRate(final double rate) {
+        return rate * this.rateFactor;
+    }
+
+    private double convertDuration(final double duration) {
+        return duration / this.durationFactor;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricScheduledThreadPoolExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricScheduledThreadPoolExecutor.java
new file mode 100644
index 0000000..5dbc289
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricScheduledThreadPoolExecutor.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+
+import com.codahale.metrics.Timer;
+
+/**
+ * A {@link java.util.concurrent.ThreadPoolExecutor} that can additionally
+ * schedule commands to run after a given delay with a timer metric
+ * which aggregates timing durations and provides duration statistics.
+ *
+ * @author jiachun.fjc
+ */
+public class MetricScheduledThreadPoolExecutor extends LogScheduledThreadPoolExecutor {
+
+    public MetricScheduledThreadPoolExecutor(int corePoolSize, String name) {
+        super(corePoolSize, name);
+    }
+
+    public MetricScheduledThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory, String name) {
+        super(corePoolSize, threadFactory, name);
+    }
+
+    public MetricScheduledThreadPoolExecutor(int corePoolSize, RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, handler, name);
+    }
+
+    public MetricScheduledThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory,
+                                             RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, threadFactory, handler, name);
+    }
+
+    @Override
+    protected void beforeExecute(Thread t, Runnable r) {
+        super.beforeExecute(t, r);
+        try {
+            ThreadPoolMetricRegistry.timerThreadLocal() //
+                .set(ThreadPoolMetricRegistry.metricRegistry().timer("scheduledThreadPool." + getName()).time());
+        } catch (final Throwable ignored) {
+            // ignored
+        }
+    }
+
+    @Override
+    protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+        try {
+            final ThreadLocal tl = ThreadPoolMetricRegistry.timerThreadLocal();
+            final Timer.Context ctx = tl.get();
+            if (ctx != null) {
+                ctx.stop();
+                tl.remove();
+            }
+        } catch (final Throwable ignored) {
+            // ignored
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricThreadPoolExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricThreadPoolExecutor.java
new file mode 100644
index 0000000..87e1d39
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/MetricThreadPoolExecutor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import com.codahale.metrics.Timer;
+
+/**
+ * A {@link java.util.concurrent.ExecutorService} that with a timer metric
+ * which aggregates timing durations and provides duration statistics.
+ *
+ * @author jiachun.fjc
+ */
+public class MetricThreadPoolExecutor extends LogThreadPoolExecutor {
+
+    public MetricThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                    BlockingQueue workQueue, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, name);
+    }
+
+    public MetricThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                    BlockingQueue workQueue, ThreadFactory threadFactory, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, name);
+    }
+
+    public MetricThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                    BlockingQueue workQueue, RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler, name);
+    }
+
+    public MetricThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                    BlockingQueue workQueue, ThreadFactory threadFactory,
+                                    RejectedExecutionHandler handler, String name) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler, name);
+    }
+
+    @Override
+    protected void beforeExecute(Thread t, Runnable r) {
+        super.beforeExecute(t, r);
+        try {
+            ThreadPoolMetricRegistry.timerThreadLocal() //
+                .set(ThreadPoolMetricRegistry.metricRegistry().timer("threadPool." + getName()).time());
+        } catch (final Throwable ignored) {
+            // ignored
+        }
+    }
+
+    @Override
+    protected void afterExecute(Runnable r, Throwable t) {
+        super.afterExecute(r, t);
+        try {
+            final ThreadLocal tl = ThreadPoolMetricRegistry.timerThreadLocal();
+            final Timer.Context ctx = tl.get();
+            if (ctx != null) {
+                ctx.stop();
+                tl.remove();
+            }
+        } catch (final Throwable ignored) {
+            // ignored
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Mpsc.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Mpsc.java
new file mode 100644
index 0000000..76a5987
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Mpsc.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.Queue;
+
+import org.jctools.queues.MpscChunkedArrayQueue;
+import org.jctools.queues.MpscUnboundedArrayQueue;
+import org.jctools.queues.atomic.MpscGrowableAtomicArrayQueue;
+import org.jctools.queues.atomic.MpscUnboundedAtomicArrayQueue;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class Mpsc {
+
+    private static final int MPSC_CHUNK_SIZE       = 1024;
+    private static final int MIN_MAX_MPSC_CAPACITY = MPSC_CHUNK_SIZE << 1;
+
+    public static Queue newMpscQueue() {
+        return UnsafeUtil.hasUnsafe() ? new MpscUnboundedArrayQueue<>(MPSC_CHUNK_SIZE)
+            : new MpscUnboundedAtomicArrayQueue<>(MPSC_CHUNK_SIZE);
+    }
+
+    public static Queue newMpscQueue(final int maxCapacity) {
+        final int capacity = Math.max(MIN_MAX_MPSC_CAPACITY, maxCapacity);
+        return UnsafeUtil.hasUnsafe() ? new MpscChunkedArrayQueue<>(MPSC_CHUNK_SIZE, capacity)
+            : new MpscGrowableAtomicArrayQueue<>(MPSC_CHUNK_SIZE, capacity);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NamedThreadFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NamedThreadFactory.java
new file mode 100644
index 0000000..fbd723f
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NamedThreadFactory.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Named thread factory with prefix.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-21 11:32:02 AM
+ */
+public class NamedThreadFactory implements ThreadFactory {
+
+    private static final Logger                      LOG                 = LoggerFactory
+                                                                             .getLogger(NamedThreadFactory.class);
+
+    private static final LogUncaughtExceptionHandler UNCAUGHT_EX_HANDLER = new LogUncaughtExceptionHandler();
+
+    private final String                             prefix;
+
+    private final AtomicInteger                      counter             = new AtomicInteger(0);
+    private final boolean                            daemon;
+
+    public NamedThreadFactory(String prefix) {
+        this(prefix, false);
+    }
+
+    public NamedThreadFactory(String prefix, boolean daemon) {
+        super();
+        this.prefix = prefix;
+        this.daemon = daemon;
+    }
+
+    @Override
+    public Thread newThread(Runnable r) {
+        Thread t = new Thread(r);
+        t.setDaemon(this.daemon);
+        t.setUncaughtExceptionHandler(UNCAUGHT_EX_HANDLER);
+        t.setName(this.prefix + counter.getAndIncrement());
+        return t;
+    }
+
+    private static final class LogUncaughtExceptionHandler implements Thread.UncaughtExceptionHandler {
+
+        @Override
+        public void uncaughtException(Thread t, Throwable e) {
+            LOG.error("Uncaught exception in thread {}", t, e);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NonReentrantLock.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NonReentrantLock.java
new file mode 100644
index 0000000..0b806c6
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/NonReentrantLock.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.AbstractQueuedSynchronizer;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+
+/**
+ * Non reentrant lock, copied from netty project.
+ */
+public final class NonReentrantLock extends AbstractQueuedSynchronizer implements Lock {
+
+    private static final long serialVersionUID = -833780837233068610L;
+
+    private Thread            owner;
+
+    @Override
+    public void lock() {
+        acquire(1);
+    }
+
+    @Override
+    public void lockInterruptibly() throws InterruptedException {
+        acquireInterruptibly(1);
+    }
+
+    @Override
+    public boolean tryLock() {
+        return tryAcquire(1);
+    }
+
+    @Override
+    public boolean tryLock(final long time, final TimeUnit unit) throws InterruptedException {
+        return tryAcquireNanos(1, unit.toNanos(time));
+    }
+
+    @Override
+    public void unlock() {
+        release(1);
+    }
+
+    public boolean isHeldByCurrentThread() {
+        return isHeldExclusively();
+    }
+
+    public Thread getOwner() {
+        return this.owner;
+    }
+
+    @Override
+    public Condition newCondition() {
+        return new ConditionObject();
+    }
+
+    @Override
+    protected boolean tryAcquire(final int acquires) {
+        if (compareAndSetState(0, 1)) {
+            this.owner = Thread.currentThread();
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    protected boolean tryRelease(final int releases) {
+        if (Thread.currentThread() != this.owner) {
+            throw new IllegalMonitorStateException("Owner is " + this.owner);
+        }
+        this.owner = null;
+        setState(0);
+        return true;
+    }
+
+    @Override
+    protected boolean isHeldExclusively() {
+        return getState() != 0 && this.owner == Thread.currentThread();
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/OnlyForTest.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/OnlyForTest.java
new file mode 100644
index 0000000..5652f0e
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/OnlyForTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import javax.annotation.concurrent.NotThreadSafe;
+
+/**
+ * The type/method/field etc. to which this annotation is applied is only for unit test.
+ * It means that user should not use them in business code except test code.
+ *
+ * @see NotThreadSafe
+ */
+@Documented
+@Target({ ElementType.TYPE, ElementType.METHOD, ElementType.FIELD, ElementType.CONSTRUCTOR })
+@Retention(RetentionPolicy.CLASS)
+public @interface OnlyForTest {
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Platform.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Platform.java
new file mode 100644
index 0000000..e226d2b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Platform.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.Locale;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Platform {
+
+    private static final Logger  LOG        = LoggerFactory.getLogger(Platform.class);
+
+    private static final boolean IS_WINDOWS = isWindows0();
+
+    private static final boolean IS_MAC     = isMac0();
+
+    /**
+     * Return {@code true} if the JVM is running on Windows
+     */
+    public static boolean isWindows() {
+        return IS_WINDOWS;
+    }
+
+    /**
+     * Return {@code true} if the JVM is running on Mac OSX
+     */
+    public static boolean isMac() {
+        return IS_MAC;
+    }
+
+    private static boolean isMac0() {
+        final boolean mac = SystemPropertyUtil.get("os.name", "") //
+            .toLowerCase(Locale.US) //
+            .contains("mac os x");
+        if (mac) {
+            LOG.debug("Platform: Mac OS X");
+        }
+        return mac;
+    }
+
+    private static boolean isWindows0() {
+        final boolean windows = SystemPropertyUtil.get("os.name", "") //
+            .toLowerCase(Locale.US) //
+            .contains("win");
+        if (windows) {
+            LOG.debug("Platform: Windows");
+        }
+        return windows;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclable.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclable.java
new file mode 100644
index 0000000..8c25a37
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface Recyclable {
+
+    /**
+     * Recycle this instance.
+     */
+    boolean recycle();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java
new file mode 100644
index 0000000..928dffd
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecyclableByteBufferList.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+
+/**
+ * A simple {@link java.nio.ByteBuffer} list which is recyclable.
+ * This implementation does not allow {@code null} elements to be added.
+ */
+public final class RecyclableByteBufferList extends ArrayList implements Recyclable {
+
+    private static final long serialVersionUID         = -8605125654176467947L;
+
+    private static final int  DEFAULT_INITIAL_CAPACITY = 8;
+
+    private int               capacity                 = 0;
+
+    /**
+     * Create a new empty {@link RecyclableByteBufferList} instance
+     */
+    public static RecyclableByteBufferList newInstance() {
+        return newInstance(DEFAULT_INITIAL_CAPACITY);
+    }
+
+    /**
+     * Create a new empty {@link RecyclableByteBufferList} instance with the given capacity.
+     */
+    public static RecyclableByteBufferList newInstance(final int minCapacity) {
+        final RecyclableByteBufferList ret = recyclers.get();
+        ret.ensureCapacity(minCapacity);
+        return ret;
+    }
+
+    public int getCapacity() {
+        return this.capacity;
+    }
+
+    @Override
+    public boolean addAll(final Collection c) {
+        throw reject("addAll");
+    }
+
+    @Override
+    public boolean addAll(final int index, final Collection c) {
+        throw reject("addAll");
+    }
+
+    @Override
+    public boolean add(final ByteBuffer element) {
+        if (element == null) {
+            throw new NullPointerException("element");
+        }
+        this.capacity += element.remaining();
+        return super.add(element);
+    }
+
+    @Override
+    public void add(final int index, final ByteBuffer element) {
+        if (element == null) {
+            throw new NullPointerException("element");
+        }
+        this.capacity += element.remaining();
+        super.add(index, element);
+    }
+
+    @Override
+    public ByteBuffer set(final int index, final ByteBuffer element) {
+        throw reject("set");
+    }
+
+    @Override
+    public ByteBuffer remove(final int index) {
+        throw reject("remove");
+    }
+
+    @Override
+    public boolean remove(final Object o) {
+        throw reject("remove");
+    }
+
+    @Override
+    public boolean recycle() {
+        clear();
+        this.capacity = 0;
+        return recyclers.recycle(this, handle);
+    }
+
+    public static int threadLocalCapacity() {
+        return recyclers.threadLocalCapacity();
+    }
+
+    public static int threadLocalSize() {
+        return recyclers.threadLocalSize();
+    }
+
+    private static UnsupportedOperationException reject(final String message) {
+        return new UnsupportedOperationException(message);
+    }
+
+    private RecyclableByteBufferList(final Recyclers.Handle handle) {
+        this(handle, DEFAULT_INITIAL_CAPACITY);
+    }
+
+    private RecyclableByteBufferList(final Recyclers.Handle handle, final int initialCapacity) {
+        super(initialCapacity);
+        this.handle = handle;
+    }
+
+    private transient final Recyclers.Handle                 handle;
+
+    private static final Recyclers recyclers = new Recyclers(512) {
+
+                                                                           @Override
+                                                                           protected RecyclableByteBufferList newObject(final Handle handle) {
+                                                                               return new RecyclableByteBufferList(
+                                                                                   handle);
+                                                                           }
+                                                                       };
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecycleUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecycleUtil.java
new file mode 100644
index 0000000..0fd9690
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RecycleUtil.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Recycle tool for {@link Recyclable}.
+ *
+ * @author jiachun.fjc
+ */
+public final class RecycleUtil {
+
+    /**
+     * Recycle designated instance.
+     */
+    public static boolean recycle(final Object obj) {
+        return obj instanceof Recyclable && ((Recyclable) obj).recycle();
+    }
+
+    private RecycleUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclers.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclers.java
new file mode 100644
index 0000000..2552070
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Recyclers.java
@@ -0,0 +1,414 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.lang.ref.WeakReference;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Light-weight object pool based on a thread-local stack.
+ * 

+ * Forked from Netty.
+ *
+ * @param  the type of the pooled object
+ */
+public abstract class Recyclers {
+
+    private static final Logger LOG = LoggerFactory.getLogger(Recyclers.class);
+
+    private static final AtomicInteger idGenerator = new AtomicInteger(Integer.MIN_VALUE);
+
+    private static final int OWN_THREAD_ID = idGenerator.getAndIncrement();
+    private static final int DEFAULT_INITIAL_MAX_CAPACITY_PER_THREAD = 4 * 1024; // Use 4k instances as default.
+    private static final int MAX_CAPACITY_PER_THREAD;
+    private static final int INITIAL_CAPACITY;
+
+    static {
+        int maxCapacityPerThread = SystemPropertyUtil.getInt("jraft.recyclers.maxCapacityPerThread", DEFAULT_INITIAL_MAX_CAPACITY_PER_THREAD);
+        if (maxCapacityPerThread < 0) {
+            maxCapacityPerThread = DEFAULT_INITIAL_MAX_CAPACITY_PER_THREAD;
+        }
+
+        MAX_CAPACITY_PER_THREAD = maxCapacityPerThread;
+
+        LOG.info("-Djraft.recyclers.maxCapacityPerThread: {}.", MAX_CAPACITY_PER_THREAD == 0 ? "disabled" : MAX_CAPACITY_PER_THREAD);
+
+        INITIAL_CAPACITY = Math.min(MAX_CAPACITY_PER_THREAD, 256);
+    }
+
+    public static final Handle NOOP_HANDLE = new Handle() {};
+
+    private final int maxCapacityPerThread;
+    private final ThreadLocal> threadLocal = new ThreadLocal>() {
+
+        @Override
+        protected Stack initialValue() {
+            return new Stack<>(Recyclers.this, Thread.currentThread(), maxCapacityPerThread);
+        }
+    };
+
+    protected Recyclers() {
+        this(MAX_CAPACITY_PER_THREAD);
+    }
+
+    protected Recyclers(int maxCapacityPerThread) {
+        this.maxCapacityPerThread = Math.min(MAX_CAPACITY_PER_THREAD, Math.max(0, maxCapacityPerThread));
+    }
+
+    @SuppressWarnings("unchecked")
+    public final T get() {
+        if (maxCapacityPerThread == 0) {
+            return newObject(NOOP_HANDLE);
+        }
+        Stack stack = threadLocal.get();
+        DefaultHandle handle = stack.pop();
+        if (handle == null) {
+            handle = stack.newHandle();
+            handle.value = newObject(handle);
+        }
+        return (T) handle.value;
+    }
+
+    public final boolean recycle(T o, Handle handle) {
+        if (handle == NOOP_HANDLE) {
+            return false;
+        }
+
+        DefaultHandle h = (DefaultHandle) handle;
+
+        final Stack stack = h.stack;
+        if (h.lastRecycledId != h.recycleId || stack == null) {
+            throw new IllegalStateException("recycled already");
+        }
+
+        if (stack.parent != this) {
+            return false;
+        }
+        if (o != h.value) {
+            throw new IllegalArgumentException("o does not belong to handle");
+        }
+        h.recycle();
+        return true;
+    }
+
+    protected abstract T newObject(Handle handle);
+
+    public final int threadLocalCapacity() {
+        return threadLocal.get().elements.length;
+    }
+
+    public final int threadLocalSize() {
+        return threadLocal.get().size;
+    }
+
+    public interface Handle {}
+
+    static final class DefaultHandle implements Handle {
+        private int lastRecycledId;
+        private int recycleId;
+
+        private Stack stack;
+        private Object value;
+
+        DefaultHandle(Stack stack) {
+            this.stack = stack;
+        }
+
+        public void recycle() {
+            Thread thread = Thread.currentThread();
+
+            final Stack stack = this.stack;
+            if (lastRecycledId != recycleId || stack == null) {
+                throw new IllegalStateException("recycled already");
+            }
+
+            if (thread == stack.thread) {
+                stack.push(this);
+                return;
+            }
+            // we don't want to have a ref to the queue as the value in our weak map
+            // so we null it out; to ensure there are no races with restoring it later
+            // we impose a memory ordering here (no-op on x86)
+            Map, WeakOrderQueue> delayedRecycled = Recyclers.delayedRecycled.get();
+            WeakOrderQueue queue = delayedRecycled.get(stack);
+            if (queue == null) {
+                delayedRecycled.put(stack, queue = new WeakOrderQueue(stack, thread));
+            }
+            queue.add(this);
+        }
+    }
+
+    private static final ThreadLocal, WeakOrderQueue>> delayedRecycled = ThreadLocal.withInitial(WeakHashMap::new);
+
+    // a queue that makes only moderate guarantees about visibility: items are seen in the correct order,
+    // but we aren't absolutely guaranteed to ever see anything at all, thereby keeping the queue cheap to maintain
+    private static final class WeakOrderQueue {
+        private static final int LINK_CAPACITY = 16;
+
+        // Let Link extend AtomicInteger for intrinsics. The Link itself will be used as writerIndex.
+        @SuppressWarnings("serial")
+        private static final class Link extends AtomicInteger {
+            private final DefaultHandle[] elements = new DefaultHandle[LINK_CAPACITY];
+
+            private int readIndex;
+            private Link next;
+        }
+
+        // chain of data items
+        private Link head, tail;
+        // pointer to another queue of delayed items for the same stack
+        private WeakOrderQueue next;
+        private final WeakReference owner;
+        private final int id = idGenerator.getAndIncrement();
+
+        WeakOrderQueue(Stack stack, Thread thread) {
+            head = tail = new Link();
+            owner = new WeakReference<>(thread);
+            synchronized (stackLock(stack)) {
+                next = stack.head;
+                stack.head = this;
+            }
+        }
+
+        private Object stackLock(Stack stack) {
+            return stack;
+        }
+
+        void add(DefaultHandle handle) {
+            handle.lastRecycledId = id;
+
+            Link tail = this.tail;
+            int writeIndex;
+            if ((writeIndex = tail.get()) == LINK_CAPACITY) {
+                this.tail = tail = tail.next = new Link();
+                writeIndex = tail.get();
+            }
+            tail.elements[writeIndex] = handle;
+            handle.stack = null;
+            // we lazy set to ensure that setting stack to null appears before we unnull it in the owning thread;
+            // this also means we guarantee visibility of an element in the queue if we see the index updated
+            tail.lazySet(writeIndex + 1);
+        }
+
+        boolean hasFinalData() {
+            return tail.readIndex != tail.get();
+        }
+
+        // transfer as many items as we can from this queue to the stack, returning true if any were transferred
+        @SuppressWarnings("rawtypes")
+        boolean transfer(Stack dst) {
+
+            Link head = this.head;
+            if (head == null) {
+                return false;
+            }
+
+            if (head.readIndex == LINK_CAPACITY) {
+                if (head.next == null) {
+                    return false;
+                }
+                this.head = head = head.next;
+            }
+
+            final int srcStart = head.readIndex;
+            int srcEnd = head.get();
+            final int srcSize = srcEnd - srcStart;
+            if (srcSize == 0) {
+                return false;
+            }
+
+            final int dstSize = dst.size;
+            final int expectedCapacity = dstSize + srcSize;
+
+            if (expectedCapacity > dst.elements.length) {
+                final int actualCapacity = dst.increaseCapacity(expectedCapacity);
+                srcEnd = Math.min(srcStart + actualCapacity - dstSize, srcEnd);
+            }
+
+            if (srcStart != srcEnd) {
+                final DefaultHandle[] srcElems = head.elements;
+                final DefaultHandle[] dstElems = dst.elements;
+                int newDstSize = dstSize;
+                for (int i = srcStart; i < srcEnd; i++) {
+                    DefaultHandle element = srcElems[i];
+                    if (element.recycleId == 0) {
+                        element.recycleId = element.lastRecycledId;
+                    } else if (element.recycleId != element.lastRecycledId) {
+                        throw new IllegalStateException("recycled already");
+                    }
+                    element.stack = dst;
+                    dstElems[newDstSize++] = element;
+                    srcElems[i] = null;
+                }
+                dst.size = newDstSize;
+
+                if (srcEnd == LINK_CAPACITY && head.next != null) {
+                    this.head = head.next;
+                }
+
+                head.readIndex = srcEnd;
+                return true;
+            } else {
+                // The destination stack is full already.
+                return false;
+            }
+        }
+    }
+
+    static final class Stack {
+
+        // we keep a queue of per-thread queues, which is appended to once only, each time a new thread other
+        // than the stack owner recycles: when we run out of items in our stack we iterate this collection
+        // to scavenge those that can be reused. this permits us to incur minimal thread synchronisation whilst
+        // still recycling all items.
+        final Recyclers parent;
+        final Thread thread;
+        private DefaultHandle[] elements;
+        private final int maxCapacity;
+        private int size;
+
+        private volatile WeakOrderQueue head;
+        private WeakOrderQueue cursor, prev;
+
+        Stack(Recyclers parent, Thread thread, int maxCapacity) {
+            this.parent = parent;
+            this.thread = thread;
+            this.maxCapacity = maxCapacity;
+            elements = new DefaultHandle[Math.min(INITIAL_CAPACITY, maxCapacity)];
+        }
+
+        int increaseCapacity(int expectedCapacity) {
+            int newCapacity = elements.length;
+            int maxCapacity = this.maxCapacity;
+            do {
+                newCapacity <<= 1;
+            } while (newCapacity < expectedCapacity && newCapacity < maxCapacity);
+
+            newCapacity = Math.min(newCapacity, maxCapacity);
+            if (newCapacity != elements.length) {
+                elements = Arrays.copyOf(elements, newCapacity);
+            }
+
+            return newCapacity;
+        }
+
+        DefaultHandle pop() {
+            int size = this.size;
+            if (size == 0) {
+                if (!scavenge()) {
+                    return null;
+                }
+                size = this.size;
+            }
+            size--;
+            DefaultHandle ret = elements[size];
+            if (ret.lastRecycledId != ret.recycleId) {
+                throw new IllegalStateException("recycled multiple times");
+            }
+            ret.recycleId = 0;
+            ret.lastRecycledId = 0;
+            this.size = size;
+            return ret;
+        }
+
+        boolean scavenge() {
+            // continue an existing scavenge, if any
+            if (scavengeSome()) {
+                return true;
+            }
+
+            // reset our scavenge cursor
+            prev = null;
+            cursor = head;
+            return false;
+        }
+
+        boolean scavengeSome() {
+            WeakOrderQueue cursor = this.cursor;
+            if (cursor == null) {
+                cursor = head;
+                if (cursor == null) {
+                    return false;
+                }
+            }
+
+            boolean success = false;
+            WeakOrderQueue prev = this.prev;
+            do {
+                if (cursor.transfer(this)) {
+                    success = true;
+                    break;
+                }
+
+                WeakOrderQueue next = cursor.next;
+                if (cursor.owner.get() == null) {
+                    // If the thread associated with the queue is gone, unlink it, after
+                    // performing a volatile read to confirm there is no data left to collect.
+                    // We never unlink the first queue, as we don't want to synchronize on updating the head.
+                    if (cursor.hasFinalData()) {
+                        for (;;) {
+                            if (cursor.transfer(this)) {
+                                success = true;
+                            } else {
+                                break;
+                            }
+                        }
+                    }
+                    if (prev != null) {
+                        prev.next = next;
+                    }
+                } else {
+                    prev = cursor;
+                }
+
+                cursor = next;
+
+            } while (cursor != null && !success);
+
+            this.prev = prev;
+            this.cursor = cursor;
+            return success;
+        }
+
+        void push(DefaultHandle item) {
+            if ((item.recycleId | item.lastRecycledId) != 0) {
+                throw new IllegalStateException("recycled already");
+            }
+            item.recycleId = item.lastRecycledId = OWN_THREAD_ID;
+
+            int size = this.size;
+            if (size >= maxCapacity) {
+                // Hit the maximum capacity - drop the possibly youngest object.
+                return;
+            }
+            if (size == elements.length) {
+                elements = Arrays.copyOf(elements, Math.min(size << 1, maxCapacity));
+            }
+
+            elements[size] = item;
+            this.size = size + 1;
+        }
+
+        DefaultHandle newHandle() {
+            return new DefaultHandle(this);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RepeatedTimer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RepeatedTimer.java
new file mode 100644
index 0000000..7b04f63
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RepeatedTimer.java
@@ -0,0 +1,295 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.util.timer.HashedWheelTimer;
+import com.alipay.sofa.jraft.util.timer.Timeout;
+import com.alipay.sofa.jraft.util.timer.Timer;
+import com.alipay.sofa.jraft.util.timer.TimerTask;
+
+/**
+ * Repeatable timer based on java.util.Timer.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-30 3:45:37 PM
+ */
+public abstract class RepeatedTimer implements Describer {
+
+    public static final Logger LOG  = LoggerFactory.getLogger(RepeatedTimer.class);
+
+    private final Lock         lock = new ReentrantLock();
+    private final Timer        timer;
+    private Timeout            timeout;
+    private boolean            stopped;
+    private volatile boolean   running;
+    private volatile boolean   destroyed;
+    private volatile boolean   invoking;
+    private volatile int       timeoutMs;
+    private final String       name;
+
+    public int getTimeoutMs() {
+        return this.timeoutMs;
+    }
+
+    public RepeatedTimer(final String name, final int timeoutMs) {
+        this(name, timeoutMs, new HashedWheelTimer(new NamedThreadFactory(name, true), 1, TimeUnit.MILLISECONDS, 2048));
+    }
+
+    public RepeatedTimer(final String name, final int timeoutMs, final Timer timer) {
+        super();
+        this.name = name;
+        this.timeoutMs = timeoutMs;
+        this.stopped = true;
+        this.timer = Requires.requireNonNull(timer, "timer");
+    }
+
+    /**
+     * Subclasses should implement this method for timer trigger.
+     */
+    protected abstract void onTrigger();
+
+    /**
+     * Adjust timeoutMs before every scheduling.
+     *
+     * @param timeoutMs timeout millis
+     * @return timeout millis
+     */
+    protected int adjustTimeout(final int timeoutMs) {
+        return timeoutMs;
+    }
+
+    public void run() {
+        this.invoking = true;
+        try {
+            onTrigger();
+        } catch (final Throwable t) {
+            LOG.error("Run timer failed.", t);
+        }
+        boolean invokeDestroyed = false;
+        this.lock.lock();
+        try {
+            this.invoking = false;
+            if (this.stopped) {
+                this.running = false;
+                invokeDestroyed = this.destroyed;
+            } else {
+                this.timeout = null;
+                schedule();
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        if (invokeDestroyed) {
+            onDestroy();
+        }
+    }
+
+    /**
+     * Run the timer at once, it will cancel the timer and re-schedule it.
+     */
+    public void runOnceNow() {
+        this.lock.lock();
+        try {
+            if (this.timeout != null && this.timeout.cancel()) {
+                this.timeout = null;
+                run();
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    /**
+     * Called after destroy timer.
+     */
+    protected void onDestroy() {
+        LOG.info("Destroy timer: {}.", this);
+    }
+
+    /**
+     * Start the timer.
+     */
+    public void start() {
+        this.lock.lock();
+        try {
+            if (this.destroyed) {
+                return;
+            }
+            if (!this.stopped) {
+                return;
+            }
+            this.stopped = false;
+            if (this.running) {
+                return;
+            }
+            this.running = true;
+            schedule();
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    /**
+     * Restart the timer.
+     * It will be started if it's stopped, and it will be restarted if it's running.
+     *
+     * @author Qing Wang (kingchin1218@gmail.com)
+     *
+     * 2020-Mar-26 20:38:37 PM
+     */
+    public void restart() {
+        this.lock.lock();
+        try {
+            if (this.destroyed) {
+                return;
+            }
+            this.stopped = false;
+            this.running = true;
+            schedule();
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    private void schedule() {
+        if (this.timeout != null) {
+            this.timeout.cancel();
+        }
+        final TimerTask timerTask = timeout -> {
+            try {
+                RepeatedTimer.this.run();
+            } catch (final Throwable t) {
+                LOG.error("Run timer task failed, taskName={}.", RepeatedTimer.this.name, t);
+            }
+        };
+        this.timeout = this.timer.newTimeout(timerTask, adjustTimeout(this.timeoutMs), TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * Reset timer with new timeoutMs.
+     *
+     * @param timeoutMs timeout millis
+     */
+    public void reset(final int timeoutMs) {
+        this.lock.lock();
+        this.timeoutMs = timeoutMs;
+        try {
+            if (this.stopped) {
+                return;
+            }
+            if (this.running) {
+                schedule();
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    /**
+     * Reset timer with current timeoutMs
+     */
+    public void reset() {
+        this.lock.lock();
+        try {
+            reset(this.timeoutMs);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    /**
+     * Destroy timer
+     */
+    public void destroy() {
+        boolean invokeDestroyed = false;
+        this.lock.lock();
+        try {
+            if (this.destroyed) {
+                return;
+            }
+            this.destroyed = true;
+            if (!this.running) {
+                invokeDestroyed = true;
+            }
+            if (this.stopped) {
+                return;
+            }
+            this.stopped = true;
+            if (this.timeout != null) {
+                if (this.timeout.cancel()) {
+                    invokeDestroyed = true;
+                    this.running = false;
+                }
+                this.timeout = null;
+            }
+        } finally {
+            this.lock.unlock();
+            this.timer.stop();
+            if (invokeDestroyed) {
+                onDestroy();
+            }
+        }
+    }
+
+    /**
+     * Stop timer
+     */
+    public void stop() {
+        this.lock.lock();
+        try {
+            if (this.stopped) {
+                return;
+            }
+            this.stopped = true;
+            if (this.timeout != null) {
+                this.timeout.cancel();
+                this.running = false;
+                this.timeout = null;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    @Override
+    public void describe(final Printer out) {
+        final String _describeString;
+        this.lock.lock();
+        try {
+            _describeString = toString();
+        } finally {
+            this.lock.unlock();
+        }
+        out.print("  ") //
+            .println(_describeString);
+    }
+
+    @Override
+    public String toString() {
+        return "RepeatedTimer{" + "timeout=" + this.timeout + ", stopped=" + this.stopped + ", running=" + this.running
+               + ", destroyed=" + this.destroyed + ", invoking=" + this.invoking + ", timeoutMs=" + this.timeoutMs
+               + ", name='" + this.name + '\'' + '}';
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Requires.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Requires.java
new file mode 100644
index 0000000..b74a228
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Requires.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+/**
+ * Simple static methods to be called at the start of your own methods to verify
+ * correct arguments and state.
+ *
+ * @author jiachun.fjc
+ */
+public final class Requires {
+
+    /**
+     * Checks that the specified object reference is not {@code null}.
+     *
+     * @param obj the object reference to check for nullity
+     * @param  the type of the reference
+     * @return {@code obj} if not {@code null}
+     * @throws NullPointerException if {@code obj} is {@code null}
+     */
+    public static  T requireNonNull(T obj) {
+        if (obj == null) {
+            throw new NullPointerException();
+        }
+        return obj;
+    }
+
+    /**
+     * Checks that the specified object reference is not {@code null} and
+     * throws a customized {@link NullPointerException} if it is.
+     *
+     * @param obj     the object reference to check for nullity
+     * @param message detail message to be used in the event that a {@code
+     *                NullPointerException} is thrown
+     * @param      the type of the reference
+     * @return {@code obj} if not {@code null}
+     * @throws NullPointerException if {@code obj} is {@code null}
+     */
+    public static  T requireNonNull(T obj, String message) {
+        if (obj == null) {
+            throw new NullPointerException(message);
+        }
+        return obj;
+    }
+
+    /**
+     * Ensures the truth of an expression involving one or more parameters
+     * to the calling method.
+     *
+     * @param expression a boolean expression
+     * @throws IllegalArgumentException if {@code expression} is false
+     */
+    public static void requireTrue(boolean expression) {
+        if (!expression) {
+            throw new IllegalArgumentException();
+        }
+    }
+
+    /**
+     * Ensures the truth of an expression involving one or more parameters
+     * to the calling method.
+     *
+     * @param expression a boolean expression
+     * @param message    the exception message to use if the check fails;
+     *                   will be converted to a string using
+     *                   {@link String#valueOf(Object)}
+     * @throws IllegalArgumentException if {@code expression} is false
+     */
+    public static void requireTrue(boolean expression, Object message) {
+        if (!expression) {
+            throw new IllegalArgumentException(String.valueOf(message));
+        }
+    }
+
+    /**
+     * Ensures the truth of an expression involving one or more parameters
+     * to the calling method.
+     *
+     * @param expression a boolean expression
+     * @param fmt        the exception message with format string
+     * @param args       arguments referenced by the format specifiers in the format
+     *                   string
+     * @throws IllegalArgumentException if {@code expression} is false
+     */
+    public static void requireTrue(boolean expression, String fmt, Object... args) {
+        if (!expression) {
+            throw new IllegalArgumentException(String.format(fmt, args));
+        }
+    }
+
+    private Requires() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RpcFactoryHelper.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RpcFactoryHelper.java
new file mode 100644
index 0000000..e035ec1
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/RpcFactoryHelper.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import com.alipay.sofa.jraft.rpc.RaftRpcFactory;
+import com.alipay.sofa.jraft.rpc.RpcResponseFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RpcFactoryHelper {
+
+    private static final RaftRpcFactory RPC_FACTORY = JRaftServiceLoader.load(RaftRpcFactory.class) //
+                                                        .first();
+
+    public static RaftRpcFactory rpcFactory() {
+        return RPC_FACTORY;
+    }
+
+    public static RpcResponseFactory responseFactory() {
+        return RPC_FACTORY.getRpcResponseFactory();
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SPI.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SPI.java
new file mode 100644
index 0000000..b21f356
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SPI.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ ElementType.TYPE })
+public @interface SPI {
+
+    String name() default "";
+
+    int priority() default 0;
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SegmentList.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SegmentList.java
new file mode 100644
index 0000000..2341f3b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SegmentList.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.function.Predicate;
+
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+
+/**
+ * A list implementation based on segments. Only supports removing elements from start or end.
+ * The list keep the elements in a segment list, every segment contains at most 128 elements.
+ *
+ *                [segment, segment, segment ...]
+ *             /                 |                    \
+ *         segment             segment              segment
+ *      [0, 1 ... 127]    [128, 129 ... 255]    [256, 257 ... 383]
+ *
+ * @author boyan(boyan@antfin.com)
+ * @since 1.3.1
+ *
+ */
+public class SegmentList {
+    private static final int             SEGMENT_SHIFT = 7;
+    public static final int              SEGMENT_SIZE  = 2 << (SEGMENT_SHIFT - 1);
+
+    private final ArrayDeque> segments;
+
+    private int                          size;
+
+    // Cached offset in first segment.
+    private int                          firstOffset;
+
+    private final boolean                recycleSegment;
+
+    /**
+     * Create a new SegmentList
+     * @param recycleSegment true to enable recycling segment, only effective in same thread.
+     */
+    public SegmentList(final boolean recycleSegment) {
+        this.segments = new ArrayDeque<>();
+        this.size = 0;
+        this.firstOffset = 0;
+        this.recycleSegment = recycleSegment;
+    }
+
+    /**
+     * A recyclable segment.
+     * @author boyan(boyan@antfin.com)
+     *
+     * @param 
+     */
+    private final static class Segment implements Recyclable {
+        private static final Recyclers> recyclers = new Recyclers>(16_382 / SEGMENT_SIZE) {
+
+                                                                 @Override
+                                                                 protected Segment newObject(final Handle handle) {
+                                                                     return new Segment<>(handle);
+                                                                 }
+                                                             };
+
+        public static Segment newInstance(final boolean recycleSegment) {
+            if (recycleSegment) {
+                return recyclers.get();
+            } else {
+                return new Segment<>();
+            }
+        }
+
+        private transient Recyclers.Handle handle;
+
+        final T[]                          elements;
+        int                                pos;     // end offset(exclusive)
+        int                                offset;  // start offset(inclusive)
+
+        Segment() {
+            this(Recyclers.NOOP_HANDLE);
+        }
+
+        @SuppressWarnings("unchecked")
+        Segment(final Recyclers.Handle handle) {
+            this.elements = (T[]) new Object[SEGMENT_SIZE];
+            this.pos = this.offset = 0;
+            this.handle = handle;
+        }
+
+        void clear() {
+            this.pos = this.offset = 0;
+            Arrays.fill(this.elements, null);
+        }
+
+        @Override
+        public boolean recycle() {
+            clear();
+            return recyclers.recycle(this, this.handle);
+        }
+
+        int cap() {
+            return SEGMENT_SIZE - this.pos;
+        }
+
+        @SuppressWarnings("SuspiciousSystemArraycopy")
+        private void addAll(final Object[] src, final int srcPos, final int len) {
+            System.arraycopy(src, srcPos, this.elements, this.pos, len);
+            this.pos += len;
+        }
+
+        boolean isReachEnd() {
+            return this.pos == SEGMENT_SIZE;
+        }
+
+        boolean isEmpty() {
+            return this.size() == 0;
+        }
+
+        void add(final T e) {
+            this.elements[this.pos++] = e;
+        }
+
+        T get(final int index) {
+            if (index >= this.pos || index < this.offset) {
+                throw new IndexOutOfBoundsException("Index=" + index + ", Offset=" + this.offset + ", Pos=" + this.pos);
+            }
+            return this.elements[index];
+        }
+
+        T peekLast() {
+            return this.elements[this.pos - 1];
+        }
+
+        int size() {
+            return this.pos - this.offset;
+        }
+
+        T peekFirst() {
+            return this.elements[this.offset];
+        }
+
+        int removeFromLastWhen(final Predicate predicate) {
+            int removed = 0;
+            for (int i = this.pos - 1; i >= this.offset; i--) {
+                T e = this.elements[i];
+                if (predicate.test(e)) {
+                    this.elements[i] = null;
+                    removed++;
+                } else {
+                    break;
+                }
+            }
+            this.pos -= removed;
+            return removed;
+        }
+
+        int removeFromFirstWhen(final Predicate predicate) {
+            int removed = 0;
+            for (int i = this.offset; i < this.pos; i++) {
+                T e = this.elements[i];
+                if (predicate.test(e)) {
+                    this.elements[i] = null;
+                    removed++;
+                } else {
+                    break;
+                }
+            }
+            this.offset += removed;
+            return removed;
+        }
+
+        int removeFromFirst(final int toIndex) {
+            int removed = 0;
+            for (int i = this.offset; i < Math.min(toIndex, this.pos); i++) {
+                this.elements[i] = null;
+                removed++;
+            }
+            this.offset += removed;
+            return removed;
+        }
+
+        @Override
+        public String toString() {
+            StringBuilder b = new StringBuilder();
+            for (int i = this.offset; i < this.pos; i++) {
+                b.append(this.elements[i]);
+                if (i != this.pos - 1) {
+                    b.append(", ");
+                }
+            }
+            return "Segment [elements=" + b.toString() + ", offset=" + this.offset + ", pos=" + this.pos + "]";
+        }
+
+    }
+
+    public T get(int index) {
+        index += this.firstOffset;
+        return this.segments.get(index >> SEGMENT_SHIFT).get(index & (SEGMENT_SIZE - 1));
+    }
+
+    public T peekLast() {
+        Segment lastSeg = getLast();
+        return lastSeg == null ? null : lastSeg.peekLast();
+    }
+
+    public T peekFirst() {
+        Segment firstSeg = getFirst();
+        return firstSeg == null ? null : firstSeg.peekFirst();
+    }
+
+    private Segment getFirst() {
+        if (!this.segments.isEmpty()) {
+            return this.segments.peekFirst();
+        }
+        return null;
+    }
+
+    @SuppressWarnings("unchecked")
+    public void add(final T e) {
+        Segment lastSeg = getLast();
+        if (lastSeg == null || lastSeg.isReachEnd()) {
+            lastSeg = (Segment) Segment.newInstance(this.recycleSegment);
+            this.segments.add(lastSeg);
+        }
+        lastSeg.add(e);
+        this.size++;
+    }
+
+    private Segment getLast() {
+        if (!this.segments.isEmpty()) {
+            return this.segments.peekLast();
+        }
+        return null;
+    }
+
+    public int size() {
+        return this.size;
+    }
+
+    public int segmentSize() {
+        return this.segments.size();
+    }
+
+    public boolean isEmpty() {
+        return this.size == 0;
+    }
+
+    /**
+     * Remove elements from first until predicate returns false.
+     *
+     * @param predicate predicate functional interface
+     */
+    public void removeFromFirstWhen(final Predicate predicate) {
+        Segment firstSeg = getFirst();
+        while (true) {
+            if (firstSeg == null) {
+                this.firstOffset = this.size = 0;
+                return;
+            }
+            int removed = firstSeg.removeFromFirstWhen(predicate);
+            if (removed == 0) {
+                break;
+            }
+            this.size -= removed;
+            this.firstOffset = firstSeg.offset;
+            if (firstSeg.isEmpty()) {
+                RecycleUtil.recycle(this.segments.pollFirst());
+                firstSeg = getFirst();
+                this.firstOffset = 0;
+            }
+        }
+    }
+
+    public void clear() {
+        while (!this.segments.isEmpty()) {
+            RecycleUtil.recycle(this.segments.pollLast());
+        }
+        this.size = this.firstOffset = 0;
+    }
+
+    /**
+     * Remove elements from last until predicate returns false.
+     *
+     * @param predicate predicate functional interface
+     */
+    public void removeFromLastWhen(final Predicate predicate) {
+        Segment lastSeg = getLast();
+        while (true) {
+            if (lastSeg == null) {
+                this.firstOffset = this.size = 0;
+                return;
+            }
+            int removed = lastSeg.removeFromLastWhen(predicate);
+            if (removed == 0) {
+                break;
+            }
+            this.size -= removed;
+            if (lastSeg.isEmpty()) {
+                RecycleUtil.recycle(this.segments.pollLast());
+                lastSeg = getLast();
+            }
+        }
+    }
+
+    /**
+     *
+     * Removes from this list all of the elements whose index is between
+     * 0, inclusive, and {@code toIndex}, exclusive.
+     * Shifts any succeeding elements to the left (reduces their index).
+     */
+    public void removeFromFirst(final int toIndex) {
+        int alignedIndex = toIndex + this.firstOffset;
+        int toSegmentIndex = alignedIndex >> SEGMENT_SHIFT;
+
+        int toIndexInSeg = alignedIndex & (SEGMENT_SIZE - 1);
+
+        if (toSegmentIndex > 0) {
+            this.segments.removeRange(0, toSegmentIndex);
+            this.size -= ((toSegmentIndex << SEGMENT_SHIFT) - this.firstOffset);
+        }
+
+        Segment firstSeg = this.getFirst();
+        if (firstSeg != null) {
+            this.size -= firstSeg.removeFromFirst(toIndexInSeg);
+            this.firstOffset = firstSeg.offset;
+            if (firstSeg.isEmpty()) {
+                RecycleUtil.recycle(this.segments.pollFirst());
+                this.firstOffset = 0;
+            }
+        } else {
+            this.firstOffset = this.size = 0;
+        }
+    }
+
+    private static final ReferenceFieldUpdater, Object[]> LIST_ARRAY_GETTER = Updaters
+                                                                                             .newReferenceFieldUpdater(
+                                                                                                 ArrayList.class,
+                                                                                                 "elementData");
+
+    @SuppressWarnings("unchecked")
+    public void addAll(final Collection coll) {
+        Object[] src = coll2Array(coll);
+
+        int srcPos = 0;
+        int srcSize = coll.size();
+
+        Segment lastSeg = getLast();
+        while (srcPos < srcSize) {
+            if (lastSeg == null || lastSeg.isReachEnd()) {
+                lastSeg = (Segment) Segment.newInstance(this.recycleSegment);
+                this.segments.add(lastSeg);
+            }
+
+            int len = Math.min(lastSeg.cap(), srcSize - srcPos);
+            lastSeg.addAll(src, srcPos, len);
+            srcPos += len;
+            this.size += len;
+        }
+    }
+
+    private Object[] coll2Array(final Collection coll) {
+        Object[] src;
+        if (coll instanceof ArrayList && UnsafeUtil.hasUnsafe()) {
+            src = LIST_ARRAY_GETTER.get((ArrayList) coll);
+        } else {
+            src = coll.toArray();
+        }
+        return src;
+    }
+
+    @Override
+    public String toString() {
+        return "SegmentList [segments=" + this.segments + ", size=" + this.size + ", firstOffset=" + this.firstOffset
+               + "]";
+    }
+
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SignalHelper.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SignalHelper.java
new file mode 100644
index 0000000..b0fbb72
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SignalHelper.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class SignalHelper {
+
+    private static final Logger         LOG             = LoggerFactory.getLogger(SignalHelper.class);
+
+    private static final SignalAccessor SIGNAL_ACCESSOR = getSignalAccessor0();
+
+    public static final String          SIG_USR2        = "USR2";
+
+    public static boolean supportSignal() {
+        return SIGNAL_ACCESSOR != null;
+    }
+
+    /**
+     * Registers user signal handlers.
+     *
+     * @param signalName a signal name
+     * @param handlers   user signal handlers
+     * @return true if support on current platform
+     */
+    public static boolean addSignal(final String signalName, final List handlers) {
+        if (SIGNAL_ACCESSOR != null) {
+            SIGNAL_ACCESSOR.addSignal(signalName, handlers);
+            return true;
+        }
+        return false;
+    }
+
+    private static SignalAccessor getSignalAccessor0() {
+        return hasSignal0() ? new SignalAccessor() : null;
+    }
+
+    private static boolean hasSignal0() {
+        try {
+            Class.forName("sun.misc.Signal");
+            return true;
+        } catch (final Throwable t) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("sun.misc.Signal: unavailable.", t);
+            }
+        }
+        return false;
+    }
+
+    private SignalHelper() {
+    }
+
+    static class SignalAccessor {
+
+        public void addSignal(final String signalName, final List handlers) {
+            final sun.misc.Signal signal = new sun.misc.Signal(signalName);
+            final SignalHandlerAdapter adapter = new SignalHandlerAdapter(signal, handlers);
+            sun.misc.Signal.handle(signal, adapter);
+        }
+    }
+
+    static class SignalHandlerAdapter implements sun.misc.SignalHandler {
+
+        private final sun.misc.Signal          target;
+        private final List handlers;
+
+        public static void addSignal(final SignalHandlerAdapter adapter) {
+            sun.misc.Signal.handle(adapter.target, adapter);
+        }
+
+        public SignalHandlerAdapter(sun.misc.Signal target, List handlers) {
+            this.target = target;
+            this.handlers = handlers;
+        }
+
+        @Override
+        public void handle(final sun.misc.Signal signal) {
+            try {
+                if (!this.target.equals(signal)) {
+                    return;
+                }
+
+                LOG.info("Handling signal {}.", signal);
+
+                for (final JRaftSignalHandler h : this.handlers) {
+                    h.handle(signal.getName());
+                }
+            } catch (final Throwable t) {
+                LOG.error("Fail to handle signal: {}.", signal, t);
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/StorageOptionsFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/StorageOptionsFactory.java
new file mode 100644
index 0000000..dfde425
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/StorageOptionsFactory.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.BloomFilter;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.CompactionStyle;
+import org.rocksdb.CompressionType;
+import org.rocksdb.DBOptions;
+import org.rocksdb.IndexType;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksObject;
+import org.rocksdb.util.SizeUnit;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class StorageOptionsFactory {
+
+    static {
+        RocksDB.loadLibrary();
+    }
+
+    private static final Map             rocksDBOptionsTable      = new ConcurrentHashMap<>();
+    private static final Map   columnFamilyOptionsTable = new ConcurrentHashMap<>();
+    private static final Map tableFormatConfigTable   = new ConcurrentHashMap<>();
+
+    /**
+     * Releases all storage options from the responsibility of freeing the
+     * underlying native C++ object.
+     *
+     * Note, that once an instance of options has been released, calling any
+     * of its functions will lead to undefined behavior.
+     */
+    public static void releaseAllOptions() {
+        for (final DBOptions opts : rocksDBOptionsTable.values()) {
+            if (opts != null) {
+                opts.close();
+            }
+        }
+        for (final ColumnFamilyOptions opts : columnFamilyOptionsTable.values()) {
+            if (opts != null) {
+                opts.close();
+            }
+        }
+    }
+
+    /**
+     * Users can register a custom rocksdb dboptions, then the related
+     * classes will get their options by the key of their own class
+     * name.  If the user does not register an options, a default options
+     * will be provided.
+     *
+     * @param cls  the key of DBOptions
+     * @param opts the DBOptions
+     */
+    public static void registerRocksDBOptions(final Class cls, final DBOptions opts) {
+        Requires.requireNonNull(cls, "cls");
+        Requires.requireNonNull(opts, "opts");
+        if (rocksDBOptionsTable.putIfAbsent(cls.getName(), opts) != null) {
+            throw new IllegalStateException("DBOptions with class key [" + cls.getName()
+                                            + "] has already been registered");
+        }
+    }
+
+    /**
+     * Get a new default DBOptions or a copy of the exist DBOptions.
+     * Users should call DBOptions#close() to release resources themselves.
+     *
+     * @param cls the key of DBOptions
+     * @return new default DBOptions or a copy of the exist DBOptions
+     */
+    public static DBOptions getRocksDBOptions(final Class cls) {
+        Requires.requireNonNull(cls, "cls");
+        DBOptions opts = rocksDBOptionsTable.get(cls.getName());
+        if (opts == null) {
+            final DBOptions newOpts = getDefaultRocksDBOptions();
+            opts = rocksDBOptionsTable.putIfAbsent(cls.getName(), newOpts);
+            if (opts == null) {
+                opts = newOpts;
+            } else {
+                newOpts.close();
+            }
+        }
+        // NOTE: This does a shallow copy, which means env, rate_limiter,
+        // sst_file_manager, info_log and other pointers will be cloned!
+        return new DBOptions(checkInvalid(opts));
+    }
+
+    public static DBOptions getDefaultRocksDBOptions() {
+        // Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
+        final DBOptions opts = new DBOptions();
+
+        // If this value is set to true, then the database will be created if it is
+        // missing during {@code RocksDB.open()}.
+        opts.setCreateIfMissing(true);
+
+        // If true, missing column families will be automatically created.
+        opts.setCreateMissingColumnFamilies(true);
+
+        // Number of open files that can be used by the DB. You may need to increase
+        // this if your database has a large working set. Value -1 means files opened
+        // are always kept open.
+        opts.setMaxOpenFiles(-1);
+
+        // To limit the num of LOG. Once LOG exceed this num, RocksDB will delete old LOG
+        // automatically.
+        opts.setKeepLogFileNum(100);
+
+        // To limit the size of WALs. Once WALs exceed this size, RocksDB will start
+        // forcing the flush of column families to allow deletion of some oldest WALs.
+        // We make it 1G as default.
+        opts.setMaxTotalWalSize(1 << 30);
+
+        // The maximum number of concurrent background compactions. The default is 1,
+        // but to fully utilize your CPU and storage you might want to increase this
+        // to approximately number of cores in the system.
+        opts.setMaxBackgroundCompactions(Math.min(Utils.cpus(), 4));
+
+        // The maximum number of concurrent flush operations. It is usually good enough
+        // to set this to 1.
+        opts.setMaxBackgroundFlushes(1);
+
+        return opts;
+    }
+
+    /**
+     * Users can register a custom rocksdb ColumnFamilyOptions, then the
+     * related classes will get their options by the key of their own class
+     * name.  If the user does not register an options, a default options
+     * will be provided.
+     *
+     * @param cls  the key of ColumnFamilyOptions
+     * @param opts the ColumnFamilyOptions
+     */
+    public static void registerRocksDBColumnFamilyOptions(final Class cls, final ColumnFamilyOptions opts) {
+        Requires.requireNonNull(cls, "cls");
+        Requires.requireNonNull(opts, "opts");
+        if (columnFamilyOptionsTable.putIfAbsent(cls.getName(), opts) != null) {
+            throw new IllegalStateException("ColumnFamilyOptions with class key [" + cls.getName()
+                                            + "] has already been registered");
+        }
+    }
+
+    /**
+     * Get a new default ColumnFamilyOptions or a copy of the exist
+     * ColumnFamilyOptions.  Users should call ColumnFamilyOptions#close()
+     * to release resources themselves.
+     *
+     * @param cls the key of ColumnFamilyOptions
+     * @return new default ColumnFamilyOptions or a copy of the exist
+     * ColumnFamilyOptions
+     */
+    public static ColumnFamilyOptions getRocksDBColumnFamilyOptions(final Class cls) {
+        Requires.requireNonNull(cls, "cls");
+        ColumnFamilyOptions opts = columnFamilyOptionsTable.get(cls.getName());
+        if (opts == null) {
+            final ColumnFamilyOptions newOpts = getDefaultRocksDBColumnFamilyOptions();
+            opts = columnFamilyOptionsTable.putIfAbsent(cls.getName(), newOpts);
+            if (opts == null) {
+                opts = newOpts;
+            } else {
+                newOpts.close();
+            }
+        }
+        // NOTE: This does a shallow copy, which means comparator, merge_operator,
+        // compaction_filter, compaction_filter_factory and other pointers will be
+        // cloned!
+        return new ColumnFamilyOptions(checkInvalid(opts));
+    }
+
+    public static ColumnFamilyOptions getDefaultRocksDBColumnFamilyOptions() {
+        final ColumnFamilyOptions opts = new ColumnFamilyOptions();
+
+        // Flushing options:
+        // write_buffer_size sets the size of a single mem_table. Once mem_table exceeds
+        // this size, it is marked immutable and a new one is created.
+        opts.setWriteBufferSize(64 * SizeUnit.MB);
+
+        // Flushing options:
+        // max_write_buffer_number sets the maximum number of mem_tables, both active
+        // and immutable.  If the active mem_table fills up and the total number of
+        // mem_tables is larger than max_write_buffer_number we stall further writes.
+        // This may happen if the flush process is slower than the write rate.
+        opts.setMaxWriteBufferNumber(3);
+
+        // Flushing options:
+        // min_write_buffer_number_to_merge is the minimum number of mem_tables to be
+        // merged before flushing to storage. For example, if this option is set to 2,
+        // immutable mem_tables are only flushed when there are two of them - a single
+        // immutable mem_table will never be flushed.  If multiple mem_tables are merged
+        // together, less data may be written to storage since two updates are merged to
+        // a single key. However, every Get() must traverse all immutable mem_tables
+        // linearly to check if the key is there. Setting this option too high may hurt
+        // read performance.
+        opts.setMinWriteBufferNumberToMerge(1);
+
+        // Level Style Compaction:
+        // level0_file_num_compaction_trigger -- Once level 0 reaches this number of
+        // files, L0->L1 compaction is triggered. We can therefore estimate level 0
+        // size in stable state as
+        // write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger.
+        opts.setLevel0FileNumCompactionTrigger(10);
+
+        // Soft limit on number of level-0 files. We start slowing down writes at this
+        // point. A value 0 means that no writing slow down will be triggered by number
+        // of files in level-0.
+        opts.setLevel0SlowdownWritesTrigger(20);
+
+        // Maximum number of level-0 files.  We stop writes at this point.
+        opts.setLevel0StopWritesTrigger(40);
+
+        // Level Style Compaction:
+        // max_bytes_for_level_base and max_bytes_for_level_multiplier
+        //  -- max_bytes_for_level_base is total size of level 1. As mentioned, we
+        // recommend that this be around the size of level 0. Each subsequent level
+        // is max_bytes_for_level_multiplier larger than previous one. The default
+        // is 10 and we do not recommend changing that.
+        opts.setMaxBytesForLevelBase(512 * SizeUnit.MB);
+
+        // Level Style Compaction:
+        // target_file_size_base and target_file_size_multiplier
+        //  -- Files in level 1 will have target_file_size_base bytes. Each next
+        // level's file size will be target_file_size_multiplier bigger than previous
+        // one. However, by default target_file_size_multiplier is 1, so files in all
+        // L1..LMax levels are equal. Increasing target_file_size_base will reduce total
+        // number of database files, which is generally a good thing. We recommend setting
+        // target_file_size_base to be max_bytes_for_level_base / 10, so that there are
+        // 10 files in level 1.
+        opts.setTargetFileSizeBase(64 * SizeUnit.MB);
+
+        // If prefix_extractor is set and memtable_prefix_bloom_size_ratio is not 0,
+        // create prefix bloom for memtable with the size of
+        // write_buffer_size * memtable_prefix_bloom_size_ratio.
+        // If it is larger than 0.25, it is santinized to 0.25.
+        opts.setMemtablePrefixBloomSizeRatio(0.125);
+
+        // Seems like the rocksDB jni for Windows doesn't come linked with any of the
+        // compression type
+        if (!Platform.isWindows()) {
+            opts.setCompressionType(CompressionType.LZ4_COMPRESSION) //
+                .setCompactionStyle(CompactionStyle.LEVEL) //
+                .optimizeLevelStyleCompaction();
+        }
+
+        // https://github.com/facebook/rocksdb/pull/5744
+        opts.setForceConsistencyChecks(true);
+
+        return opts;
+    }
+
+    /**
+     * Users can register a custom rocksdb BlockBasedTableConfig, then the related
+     * classes will get their options by the key of their own class name.  If
+     * the user does not register a config, a default config will be provided.
+     *
+     * @param cls the key of BlockBasedTableConfig
+     * @param cfg the BlockBasedTableConfig
+     */
+    public static void registerRocksDBTableFormatConfig(final Class cls, final BlockBasedTableConfig cfg) {
+        Requires.requireNonNull(cls, "cls");
+        Requires.requireNonNull(cfg, "cfg");
+        if (tableFormatConfigTable.putIfAbsent(cls.getName(), cfg) != null) {
+            throw new IllegalStateException("TableFormatConfig with class key [" + cls.getName()
+                                            + "] has already been registered");
+        }
+    }
+
+    /**
+     * Get a new default TableFormatConfig or a copy of the exist ableFormatConfig.
+     *
+     * @param cls the key of TableFormatConfig
+     * @return new default TableFormatConfig or a copy of the exist TableFormatConfig
+     */
+    public static BlockBasedTableConfig getRocksDBTableFormatConfig(final Class cls) {
+        Requires.requireNonNull(cls, "cls");
+        BlockBasedTableConfig cfg = tableFormatConfigTable.get(cls.getName());
+        if (cfg == null) {
+            final BlockBasedTableConfig newCfg = getDefaultRocksDBTableConfig();
+            cfg = tableFormatConfigTable.putIfAbsent(cls.getName(), newCfg);
+            if (cfg == null) {
+                cfg = newCfg;
+            }
+        }
+        return copyTableFormatConfig(cfg);
+    }
+
+    public static BlockBasedTableConfig getDefaultRocksDBTableConfig() {
+        // See https://github.com/sofastack/sofa-jraft/pull/156
+        return new BlockBasedTableConfig() //
+            // Begin to use partitioned index filters
+            // https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters#how-to-use-it
+            .setIndexType(IndexType.kTwoLevelIndexSearch) //
+            .setFilterPolicy(new BloomFilter((double) 16, false)) //
+            .setPartitionFilters(true) //
+            .setMetadataBlockSize(8 * SizeUnit.KB) //
+            .setCacheIndexAndFilterBlocks(false) //
+            .setCacheIndexAndFilterBlocksWithHighPriority(true) //
+            .setPinL0FilterAndIndexBlocksInCache(true) //
+            // End of partitioned index filters settings.
+            .setBlockSize(4 * SizeUnit.KB)//
+            .setBlockCacheSize(512 * SizeUnit.MB) //
+            .setCacheNumShardBits(8);
+    }
+
+    private static BlockBasedTableConfig copyTableFormatConfig(final BlockBasedTableConfig cfg) {
+        return new BlockBasedTableConfig() //
+            .setNoBlockCache(cfg.noBlockCache()) //
+            .setBlockCacheSize(cfg.blockCacheSize()) //
+            .setCacheNumShardBits(cfg.cacheNumShardBits()) //
+            .setBlockSize(cfg.blockSize()) //
+            .setBlockSizeDeviation(cfg.blockSizeDeviation()) //
+            .setBlockRestartInterval(cfg.blockRestartInterval()) //
+            .setWholeKeyFiltering(cfg.wholeKeyFiltering()) //
+            .setCacheIndexAndFilterBlocks(cfg.cacheIndexAndFilterBlocks()) //
+            .setCacheIndexAndFilterBlocksWithHighPriority(cfg.cacheIndexAndFilterBlocksWithHighPriority()) //
+            .setPinL0FilterAndIndexBlocksInCache(cfg.pinL0FilterAndIndexBlocksInCache()) //
+            .setPartitionFilters(cfg.partitionFilters()) //
+            .setMetadataBlockSize(cfg.metadataBlockSize()) //
+            .setPinTopLevelIndexAndFilter(cfg.pinTopLevelIndexAndFilter()) //
+            .setHashIndexAllowCollision(cfg.hashIndexAllowCollision()) //
+            .setBlockCacheCompressedSize(cfg.blockCacheCompressedSize()) //
+            .setBlockCacheCompressedNumShardBits(cfg.blockCacheCompressedNumShardBits()) //
+            .setChecksumType(cfg.checksumType()) //
+            .setIndexType(cfg.indexType()) //
+            .setFormatVersion(cfg.formatVersion());
+    }
+
+    private static  T checkInvalid(final T opts) {
+        if (!opts.isOwningHandle()) {
+            throw new IllegalStateException(
+                "the instance of options [" + opts
+                        + "] has been released, calling any of its functions will lead to undefined behavior.");
+        }
+        return opts;
+    }
+
+    private StorageOptionsFactory() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SystemPropertyUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SystemPropertyUtil.java
new file mode 100644
index 0000000..2528114
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/SystemPropertyUtil.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A collection of utility methods to retrieve and
+ * parse the values of the Java system properties.
+ *
+ */
+public final class SystemPropertyUtil {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SystemPropertyUtil.class);
+
+    /**
+     * Returns {@code true} if and only if the system property
+     * with the specified {@code key} exists.
+     */
+    public static boolean contains(String key) {
+        return get(key) != null;
+    }
+
+    /**
+     * Returns the value of the Java system property with the
+     * specified {@code key}, while falling back to {@code null}
+     * if the property access fails.
+     *
+     * @return the property value or {@code null}
+     */
+    public static String get(String key) {
+        return get(key, null);
+    }
+
+    /**
+     * Returns the value of the Java system property with the
+     * specified {@code key}, while falling back to the specified
+     * default value if the property access fails.
+     *
+     * @return the property value.
+     * {@code def} if there's no such property or if an access to
+     * the specified property is not allowed.
+     */
+    public static String get(final String key, String def) {
+        if (key == null) {
+            throw new NullPointerException("key");
+        }
+        if (key.isEmpty()) {
+            throw new IllegalArgumentException("key must not be empty.");
+        }
+
+        String value = null;
+        try {
+            if (System.getSecurityManager() == null) {
+                value = System.getProperty(key);
+            } else {
+                value = AccessController.doPrivileged((PrivilegedAction) () -> System.getProperty(key));
+            }
+        } catch (Exception e) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("Unable to retrieve a system property '{}'; default values will be used, {}.", key, e);
+            }
+        }
+
+        if (value == null) {
+            return def;
+        }
+
+        return value;
+    }
+
+    /**
+     * Returns the value of the Java system property with the
+     * specified {@code key}, while falling back to the specified
+     * default value if the property access fails.
+     *
+     * @return the property value.
+     * {@code def} if there's no such property or if an access to
+     * the specified property is not allowed.
+     */
+    public static boolean getBoolean(String key, boolean def) {
+        String value = get(key);
+        if (value == null) {
+            return def;
+        }
+
+        value = value.trim().toLowerCase();
+        if (value.isEmpty()) {
+            return true;
+        }
+
+        if ("true".equals(value) || "yes".equals(value) || "1".equals(value)) {
+            return true;
+        }
+
+        if ("false".equals(value) || "no".equals(value) || "0".equals(value)) {
+            return false;
+        }
+
+        LOG.warn("Unable to parse the boolean system property '{}':{} - using the default value: {}.", key, value, def);
+
+        return def;
+    }
+
+    /**
+     * Returns the value of the Java system property with the
+     * specified {@code key}, while falling back to the specified
+     * default value if the property access fails.
+     *
+     * @return the property value.
+     * {@code def} if there's no such property or if an access
+     * to the specified property is not allowed.
+     */
+    public static int getInt(String key, int def) {
+        String value = get(key);
+        if (value == null) {
+            return def;
+        }
+
+        value = value.trim().toLowerCase();
+        try {
+            return Integer.parseInt(value);
+        } catch (Exception ignored) {
+            // ignored
+        }
+
+        LOG.warn("Unable to parse the integer system property '{}':{} - using the default value: {}.", key, value, def);
+
+        return def;
+    }
+
+    /**
+     * Returns the value of the Java system property with the
+     * specified {@code key}, while falling back to the specified
+     * default value if the property access fails.
+     *
+     * @return the property value.
+     * {@code def} if there's no such property or if an access to
+     * the specified property is not allowed.
+     */
+    public static long getLong(String key, long def) {
+        String value = get(key);
+        if (value == null) {
+            return def;
+        }
+
+        value = value.trim().toLowerCase();
+        try {
+            return Long.parseLong(value);
+        } catch (Exception ignored) {
+            // ignored
+        }
+
+        LOG.warn("Unable to parse the long integer system property '{}':{} - using the default value: {}.", key, value,
+            def);
+
+        return def;
+    }
+
+    /**
+     * Sets the value of the Java system property with the
+     * specified {@code key}
+     */
+    public static Object setProperty(String key, String value) {
+        return System.getProperties().setProperty(key, value);
+    }
+
+    private SystemPropertyUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadHelper.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadHelper.java
new file mode 100644
index 0000000..f18d60b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadHelper.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import org.objectweb.asm.ClassWriter;
+import org.objectweb.asm.MethodVisitor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.objectweb.asm.Opcodes.ACC_PUBLIC;
+import static org.objectweb.asm.Opcodes.ACC_SUPER;
+import static org.objectweb.asm.Opcodes.ACC_VARARGS;
+import static org.objectweb.asm.Opcodes.ALOAD;
+import static org.objectweb.asm.Opcodes.INVOKESPECIAL;
+import static org.objectweb.asm.Opcodes.INVOKESTATIC;
+import static org.objectweb.asm.Opcodes.RETURN;
+import static org.objectweb.asm.Opcodes.V1_1;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ThreadHelper {
+
+    private static final Logger  LOG = LoggerFactory.getLogger(ThreadHelper.class);
+
+    private static final Spinner SPINNER;
+
+    static {
+        final Object maybeException = AccessController.doPrivileged((PrivilegedAction) () -> {
+            try {
+                // noinspection JavaReflectionMemberAccess
+                Thread.class.getDeclaredMethod("onSpinWait");
+                return null;
+            } catch (final NoSuchMethodException | SecurityException e) {
+                return e;
+            }
+        });
+        if (maybeException == null) {
+            SPINNER = createSpinner();
+        } else {
+            SPINNER = new DefaultSpinner();
+        }
+    }
+
+    public static void onSpinWait() {
+        SPINNER.onSpinWait();
+    }
+
+    private ThreadHelper() {
+    }
+
+    public static abstract class Spinner {
+
+        public abstract void onSpinWait();
+    }
+
+    static class DefaultSpinner extends Spinner {
+
+        @Override
+        public void onSpinWait() {
+            Thread.yield();
+        }
+    }
+
+    private static Spinner createSpinner() {
+        final String superClassName = Spinner.class.getName();
+        final String superClassNameInternal = superClassName.replace('.', '/');
+
+        final String spinnerClassName = superClassName + "Impl";
+        final String spinnerClassNameInternal = spinnerClassName.replace('.', '/');
+
+        final String threadClassNameInternal = Thread.class.getName().replace('.', '/');
+
+        final ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_MAXS);
+        cw.visit(V1_1, ACC_PUBLIC + ACC_SUPER, spinnerClassNameInternal, null, superClassNameInternal, null);
+
+        MethodVisitor mv;
+
+        // default constructor
+        {
+            mv = cw.visitMethod(ACC_PUBLIC, "", "()V", null, null);
+            mv.visitCode();
+            mv.visitVarInsn(ALOAD, 0);
+            mv.visitMethodInsn(INVOKESPECIAL, superClassNameInternal, "", "()V", false);
+            mv.visitInsn(RETURN);
+            mv.visitMaxs(0, 0);
+            mv.visitEnd();
+        }
+
+        // implementation of method: `public abstract void onSpinWait()`
+        {
+            mv = cw.visitMethod(ACC_PUBLIC + ACC_VARARGS, "onSpinWait", "()V", null, null);
+            mv.visitCode();
+            mv.visitMethodInsn(INVOKESTATIC, threadClassNameInternal, "onSpinWait", "()V", false);
+            mv.visitInsn(RETURN);
+            mv.visitMaxs(0, 0);
+            mv.visitEnd();
+        }
+
+        cw.visitEnd();
+
+        try {
+            final byte[] classBytes = cw.toByteArray();
+            final Class spinnerClass = SpinnerClassLoader.INSTANCE.defineClass(spinnerClassName, classBytes);
+            return (Spinner) spinnerClass.getDeclaredConstructor().newInstance();
+        } catch (final Throwable t) {
+            LOG.warn("Error constructing spinner class: {}, will return a default spinner.", spinnerClassName, t);
+            return new DefaultSpinner();
+        }
+    }
+
+    private static class SpinnerClassLoader extends ClassLoader {
+
+        static final SpinnerClassLoader INSTANCE;
+
+        static {
+            ClassLoader parent = Spinner.class.getClassLoader();
+            if (parent == null) {
+                parent = ClassLoader.getSystemClassLoader();
+            }
+            INSTANCE = new SpinnerClassLoader(parent);
+        }
+
+        SpinnerClassLoader(ClassLoader parent) {
+            super(parent);
+        }
+
+        Class defineClass(final String name, final byte[] bytes) throws ClassFormatError {
+            return defineClass(name, bytes, 0, bytes.length, getClass().getProtectionDomain());
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadId.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadId.java
new file mode 100644
index 0000000..e63a155
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadId.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.locks.ReentrantLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Replicator id with lock.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Mar-29 10:59:47 AM
+ */
+public class ThreadId {
+
+    private static final Logger LOG  = LoggerFactory.getLogger(ThreadId.class);
+    private final Object        data;
+    private final ReentrantLock lock = new ReentrantLock();
+    private final OnError       onError;
+    private volatile boolean    destroyed;
+
+    /**
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Mar-29 11:01:54 AM
+     */
+    public interface OnError {
+
+        /**
+         * Error callback, it will be called in lock.
+         *
+         * @param id        the thread id
+         * @param data      the data
+         * @param errorCode the error code
+         */
+        void onError(final ThreadId id, final Object data, final int errorCode);
+    }
+
+    public ThreadId(final Object data, final OnError onError) {
+        super();
+        this.data = data;
+        this.onError = onError;
+        this.destroyed = false;
+    }
+
+    public Object getData() {
+        return this.data;
+    }
+
+    public Object lock() {
+        if (this.destroyed) {
+            return null;
+        }
+        this.lock.lock();
+        // Got the lock, double checking state.
+        if (this.destroyed) {
+            // should release lock
+            this.lock.unlock();
+            return null;
+        }
+        return this.data;
+    }
+
+    public void unlock() {
+        if (!this.lock.isHeldByCurrentThread()) {
+            LOG.warn("Fail to unlock with {}, the lock is not held by current thread {}.", this.data,
+                Thread.currentThread());
+            return;
+        }
+        this.lock.unlock();
+    }
+
+    public void join() {
+        while (!this.destroyed) {
+            ThreadHelper.onSpinWait();
+        }
+    }
+
+    @Override
+    public String toString() {
+        return this.data.toString();
+    }
+
+    public void unlockAndDestroy() {
+        if (this.destroyed) {
+            return;
+        }
+        this.destroyed = true;
+        unlock();
+    }
+
+    /**
+     * Set error code, run the onError callback
+     * with code immediately in lock.
+     * @param errorCode error code
+     */
+    public void setError(final int errorCode) {
+        if (this.destroyed) {
+            LOG.warn("ThreadId: {} already destroyed, ignore error code: {}", this.data, errorCode);
+            return;
+        }
+        this.lock.lock();
+        try {
+            if (this.destroyed) {
+                LOG.warn("ThreadId: {} already destroyed, ignore error code: {}", this.data, errorCode);
+                return;
+            }
+            if (this.onError != null) {
+                this.onError.onError(this, this.data, errorCode);
+            }
+
+        } finally {
+            // Maybe destroyed in callback
+            if (!this.destroyed) {
+                this.lock.unlock();
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricRegistry.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricRegistry.java
new file mode 100644
index 0000000..f3c916b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricRegistry.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ThreadPoolMetricRegistry {
+
+    private static final MetricRegistry             metricRegistry   = new MetricRegistry();
+    private static final ThreadLocal timerThreadLocal = new ThreadLocal<>();
+
+    /**
+     * Return the global registry of metric instances.
+     */
+    public static MetricRegistry metricRegistry() {
+        return metricRegistry;
+    }
+
+    public static ThreadLocal timerThreadLocal() {
+        return timerThreadLocal;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricSet.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricSet.java
new file mode 100644
index 0000000..224c526
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolMetricSet.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+
+/**
+ * Thread pool metric set including pool-size, queued, active, completed etc.
+ */
+public final class ThreadPoolMetricSet implements MetricSet {
+
+    private final ThreadPoolExecutor executor;
+
+    public ThreadPoolMetricSet(ThreadPoolExecutor rpcExecutor) {
+        super();
+        this.executor = rpcExecutor;
+    }
+
+    /**
+     * Return thread pool metrics
+     * @return thread pool metrics map
+     */
+    @Override
+    public Map getMetrics() {
+        final Map gauges = new HashMap<>();
+        gauges.put("pool-size", (Gauge) executor::getPoolSize);
+        gauges.put("queued", (Gauge) executor.getQueue()::size);
+        gauges.put("active", (Gauge) executor::getActiveCount);
+        gauges.put("completed", (Gauge) executor::getCompletedTaskCount);
+        return gauges;
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolUtil.java
new file mode 100644
index 0000000..2124fed
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/ThreadPoolUtil.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ThreadPoolUtil {
+
+    /**
+     * The default rejected execution handler
+     */
+    private static final RejectedExecutionHandler defaultHandler = new ThreadPoolExecutor.AbortPolicy();
+
+    public static PoolBuilder newBuilder() {
+        return new PoolBuilder();
+    }
+
+    public static ScheduledPoolBuilder newScheduledBuilder() {
+        return new ScheduledPoolBuilder();
+    }
+
+    /**
+     * Creates a new {@code MetricThreadPoolExecutor} or {@code LogThreadPoolExecutor}
+     * with the given initial parameters and default rejected execution handler.
+     *
+     * @param poolName         the name of the thread pool
+     * @param enableMetric     if metric is enabled
+     * @param coreThreads      the number of threads to keep in the pool, even if they are
+     *                         idle, unless {@code allowCoreThreadTimeOut} is set.
+     * @param maximumThreads   the maximum number of threads to allow in the pool
+     * @param keepAliveSeconds when the number of threads is greater than the core, this
+     *                         is the maximum time (seconds) that excess idle threads will
+     *                         wait for new tasks before terminating.
+     * @param workQueue        the queue to use for holding tasks before they are executed.
+     *                         This queue will hold only the {@code Runnable} tasks submitted
+     *                         by the {@code execute} method.
+     * @param threadFactory    the factory to use when the executor creates a new thread
+     * @throws IllegalArgumentException if one of the following holds:

+     *         {@code corePoolSize < 0}

+     *         {@code keepAliveSeconds < 0}

+     *         {@code maximumPoolSize <= 0}

+     *         {@code maximumPoolSize < corePoolSize}
+     * @throws NullPointerException if {@code workQueue}
+     *         or {@code threadFactory} or {@code handler} is null
+     */
+    public static ThreadPoolExecutor newThreadPool(final String poolName, final boolean enableMetric,
+                                                   final int coreThreads, final int maximumThreads,
+                                                   final long keepAliveSeconds,
+                                                   final BlockingQueue workQueue,
+                                                   final ThreadFactory threadFactory) {
+        return newThreadPool(poolName, enableMetric, coreThreads, maximumThreads, keepAliveSeconds, workQueue,
+            threadFactory, defaultHandler);
+    }
+
+    /**
+     * Creates a new {@code MetricThreadPoolExecutor} or {@code LogThreadPoolExecutor}
+     * with the given initial parameters.
+     *
+     * @param poolName         the name of the thread pool
+     * @param enableMetric     if metric is enabled
+     * @param coreThreads      the number of threads to keep in the pool, even if they are
+     *                         idle, unless {@code allowCoreThreadTimeOut} is set.
+     * @param maximumThreads   the maximum number of threads to allow in the pool
+     * @param keepAliveSeconds when the number of threads is greater than the core, this
+     *                         is the maximum time (seconds) that excess idle threads will
+     *                         wait for new tasks before terminating.
+     * @param workQueue        the queue to use for holding tasks before they are executed.
+     *                         This queue will hold only the {@code Runnable} tasks submitted
+     *                         by the {@code execute} method.
+     * @param threadFactory    the factory to use when the executor creates a new thread
+     * @param rejectedHandler  the handler to use when execution is blocked because the
+     *                         thread bounds and queue capacities are reached
+     * @throws IllegalArgumentException if one of the following holds:

+     *         {@code corePoolSize < 0}

+     *         {@code keepAliveSeconds < 0}

+     *         {@code maximumPoolSize <= 0}

+     *         {@code maximumPoolSize < corePoolSize}
+     * @throws NullPointerException if {@code workQueue}
+     *         or {@code threadFactory} or {@code handler} is null
+     */
+    public static ThreadPoolExecutor newThreadPool(final String poolName, final boolean enableMetric,
+                                                   final int coreThreads, final int maximumThreads,
+                                                   final long keepAliveSeconds,
+                                                   final BlockingQueue workQueue,
+                                                   final ThreadFactory threadFactory,
+                                                   final RejectedExecutionHandler rejectedHandler) {
+        final TimeUnit unit = TimeUnit.SECONDS;
+        if (enableMetric) {
+            return new MetricThreadPoolExecutor(coreThreads, maximumThreads, keepAliveSeconds, unit, workQueue,
+                threadFactory, rejectedHandler, poolName);
+        } else {
+            return new LogThreadPoolExecutor(coreThreads, maximumThreads, keepAliveSeconds, unit, workQueue,
+                threadFactory, rejectedHandler, poolName);
+        }
+    }
+
+    /**
+     * Creates a new ScheduledThreadPoolExecutor with the given
+     * initial parameters.
+     *
+     * @param poolName        the name of the thread pool
+     * @param enableMetric    if metric is enabled
+     * @param coreThreads     the number of threads to keep in the pool, even if they are
+     *                        idle, unless {@code allowCoreThreadTimeOut} is set.
+     * @param threadFactory   the factory to use when the executor
+     *                        creates a new thread
+     *
+     * @throws IllegalArgumentException if {@code corePoolSize < 0}
+     * @throws NullPointerException if {@code threadFactory} or
+     *         {@code handler} is null
+     * @return a new ScheduledThreadPoolExecutor
+     */
+    public static ScheduledThreadPoolExecutor newScheduledThreadPool(final String poolName, final boolean enableMetric,
+                                                                     final int coreThreads,
+                                                                     final ThreadFactory threadFactory) {
+        return newScheduledThreadPool(poolName, enableMetric, coreThreads, threadFactory, defaultHandler);
+    }
+
+    /**
+     * Creates a new ScheduledThreadPoolExecutor with the given
+     * initial parameters.
+     *
+     * @param poolName        the name of the thread pool
+     * @param enableMetric    if metric is enabled
+     * @param coreThreads     the number of threads to keep in the pool, even if they are
+     *                        idle, unless {@code allowCoreThreadTimeOut} is set.
+     * @param threadFactory   the factory to use when the executor
+     *                        creates a new thread
+     * @param rejectedHandler the handler to use when execution is blocked because the
+     *                        thread bounds and queue capacities are reached
+     *
+     * @throws IllegalArgumentException if {@code corePoolSize < 0}
+     * @throws NullPointerException if {@code threadFactory} or
+     *         {@code handler} is null
+     * @return a new ScheduledThreadPoolExecutor
+     */
+    public static ScheduledThreadPoolExecutor newScheduledThreadPool(final String poolName, final boolean enableMetric,
+                                                                     final int coreThreads,
+                                                                     final ThreadFactory threadFactory,
+                                                                     final RejectedExecutionHandler rejectedHandler) {
+        if (enableMetric) {
+            return new MetricScheduledThreadPoolExecutor(coreThreads, threadFactory, rejectedHandler, poolName);
+        } else {
+            return new LogScheduledThreadPoolExecutor(coreThreads, threadFactory, rejectedHandler, poolName);
+        }
+    }
+
+    private ThreadPoolUtil() {
+    }
+
+    public static class PoolBuilder {
+        private String                   poolName;
+        private Boolean                  enableMetric;
+        private Integer                  coreThreads;
+        private Integer                  maximumThreads;
+        private Long                     keepAliveSeconds;
+        private BlockingQueue  workQueue;
+        private ThreadFactory            threadFactory;
+        private RejectedExecutionHandler handler = ThreadPoolUtil.defaultHandler;
+
+        public PoolBuilder poolName(final String poolName) {
+            this.poolName = poolName;
+            return this;
+        }
+
+        public PoolBuilder enableMetric(final Boolean enableMetric) {
+            this.enableMetric = enableMetric;
+            return this;
+        }
+
+        public PoolBuilder coreThreads(final Integer coreThreads) {
+            this.coreThreads = coreThreads;
+            return this;
+        }
+
+        public PoolBuilder maximumThreads(final Integer maximumThreads) {
+            this.maximumThreads = maximumThreads;
+            return this;
+        }
+
+        public PoolBuilder keepAliveSeconds(final Long keepAliveSeconds) {
+            this.keepAliveSeconds = keepAliveSeconds;
+            return this;
+        }
+
+        public PoolBuilder workQueue(final BlockingQueue workQueue) {
+            this.workQueue = workQueue;
+            return this;
+        }
+
+        public PoolBuilder threadFactory(final ThreadFactory threadFactory) {
+            this.threadFactory = threadFactory;
+            return this;
+        }
+
+        public PoolBuilder rejectedHandler(final RejectedExecutionHandler handler) {
+            this.handler = handler;
+            return this;
+        }
+
+        public ThreadPoolExecutor build() {
+            Requires.requireNonNull(this.poolName, "poolName");
+            Requires.requireNonNull(this.enableMetric, "enableMetric");
+            Requires.requireNonNull(this.coreThreads, "coreThreads");
+            Requires.requireNonNull(this.maximumThreads, "maximumThreads");
+            Requires.requireNonNull(this.keepAliveSeconds, "keepAliveSeconds");
+            Requires.requireNonNull(this.workQueue, "workQueue");
+            Requires.requireNonNull(this.threadFactory, "threadFactory");
+            Requires.requireNonNull(this.handler, "handler");
+
+            return ThreadPoolUtil.newThreadPool(this.poolName, this.enableMetric, this.coreThreads,
+                this.maximumThreads, this.keepAliveSeconds, this.workQueue, this.threadFactory, this.handler);
+        }
+    }
+
+    public static class ScheduledPoolBuilder {
+        private String                   poolName;
+        private Boolean                  enableMetric;
+        private Integer                  coreThreads;
+        private ThreadFactory            threadFactory;
+        private RejectedExecutionHandler handler = ThreadPoolUtil.defaultHandler;
+
+        public ScheduledPoolBuilder poolName(final String poolName) {
+            this.poolName = poolName;
+            return this;
+        }
+
+        public ScheduledPoolBuilder enableMetric(final Boolean enableMetric) {
+            this.enableMetric = enableMetric;
+            return this;
+        }
+
+        public ScheduledPoolBuilder coreThreads(final Integer coreThreads) {
+            this.coreThreads = coreThreads;
+            return this;
+        }
+
+        public ScheduledPoolBuilder threadFactory(final ThreadFactory threadFactory) {
+            this.threadFactory = threadFactory;
+            return this;
+        }
+
+        public ScheduledPoolBuilder rejectedHandler(final RejectedExecutionHandler handler) {
+            this.handler = handler;
+            return this;
+        }
+
+        public ScheduledThreadPoolExecutor build() {
+            Requires.requireNonNull(this.poolName, "poolName");
+            Requires.requireNonNull(this.enableMetric, "enableMetric");
+            Requires.requireNonNull(this.coreThreads, "coreThreads");
+
+            Requires.requireNonNull(this.threadFactory, "threadFactory");
+            Requires.requireNonNull(this.handler, "handler");
+
+            return ThreadPoolUtil.newScheduledThreadPool(this.poolName, this.enableMetric, this.coreThreads,
+                this.threadFactory, this.handler);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Utils.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Utils.java
new file mode 100644
index 0000000..c0e30da
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/Utils.java
@@ -0,0 +1,500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import java.net.InetAddress;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.Future;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.codahale.metrics.MetricRegistry;
+
+/**
+ * Helper methods for jraft.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-07 10:12:35 AM
+ */
+public final class Utils {
+
+    private static final Logger       LOG                                 = LoggerFactory.getLogger(Utils.class);
+
+    /**
+     * The configured number of available processors. The default is
+     * {@link Runtime#availableProcessors()}. This can be overridden by setting the system property
+     * "jraft.available_processors".
+     */
+    private static final int          CPUS                                = SystemPropertyUtil.getInt(
+                                                                              "jraft.available_processors", Runtime
+                                                                                  .getRuntime().availableProcessors());
+
+    /**
+     * Default jraft closure executor pool minimum size, CPUs by default.
+     */
+    public static final int           MIN_CLOSURE_EXECUTOR_POOL_SIZE      = SystemPropertyUtil.getInt(
+                                                                              "jraft.closure.threadpool.size.min",
+                                                                              cpus());
+
+    /**
+     * Default jraft closure executor pool maximum size.
+     */
+    public static final int           MAX_CLOSURE_EXECUTOR_POOL_SIZE      = SystemPropertyUtil.getInt(
+                                                                              "jraft.closure.threadpool.size.max",
+                                                                              Math.max(100, cpus() * 5));
+
+    /**
+     * Default jraft append-entries executor(send) pool size.
+     */
+    public static final int           APPEND_ENTRIES_THREADS_SEND         = SystemPropertyUtil
+                                                                              .getInt(
+                                                                                  "jraft.append.entries.threads.send",
+                                                                                  Math.max(
+                                                                                      16,
+                                                                                      Ints.findNextPositivePowerOfTwo(cpus() * 2)));
+
+    /**
+     * Default jraft max pending tasks of append-entries per thread, 65536 by default.
+     */
+    public static final int           MAX_APPEND_ENTRIES_TASKS_PER_THREAD = SystemPropertyUtil
+                                                                              .getInt(
+                                                                                  "jraft.max.append.entries.tasks.per.thread",
+                                                                                  32768);
+
+    /**
+     * Whether use {@link com.alipay.sofa.jraft.util.concurrent.MpscSingleThreadExecutor}, true by
+     * default.
+     */
+    public static final boolean       USE_MPSC_SINGLE_THREAD_EXECUTOR     = SystemPropertyUtil.getBoolean(
+                                                                              "jraft.use.mpsc.single.thread.executor",
+                                                                              true);
+
+    /**
+     * Global thread pool to run closure.
+     */
+    private static ThreadPoolExecutor CLOSURE_EXECUTOR                    = ThreadPoolUtil
+                                                                              .newBuilder()
+                                                                              .poolName("JRAFT_CLOSURE_EXECUTOR")
+                                                                              .enableMetric(true)
+                                                                              .coreThreads(
+                                                                                  MIN_CLOSURE_EXECUTOR_POOL_SIZE)
+                                                                              .maximumThreads(
+                                                                                  MAX_CLOSURE_EXECUTOR_POOL_SIZE)
+                                                                              .keepAliveSeconds(60L)
+                                                                              .workQueue(new SynchronousQueue<>())
+                                                                              .threadFactory(
+                                                                                  new NamedThreadFactory(
+                                                                                      "JRaft-Closure-Executor-", true))
+                                                                              .build();
+
+    private static final Pattern      GROUP_ID_PATTER                     = Pattern
+                                                                              .compile("^[a-zA-Z][a-zA-Z0-9\\-_]*$");
+
+    public static void verifyGroupId(final String groupId) {
+        if (StringUtils.isBlank(groupId)) {
+            throw new IllegalArgumentException("Blank groupId");
+        }
+        if (!GROUP_ID_PATTER.matcher(groupId).matches()) {
+            throw new IllegalArgumentException(
+                "Invalid group id, it should be started with character 'a'-'z' or 'A'-'Z',"
+                        + " and followed with numbers, english alphabet, '-' or '_'. ");
+        }
+    }
+
+    /**
+     * Register CLOSURE_EXECUTOR into metric registry.
+     */
+    public static void registerClosureExecutorMetrics(final MetricRegistry registry) {
+        registry.register("raft-utils-closure-thread-pool", new ThreadPoolMetricSet(CLOSURE_EXECUTOR));
+    }
+
+    /**
+     * Run closure in current thread.
+     * @param done
+     * @param status
+     */
+    public static void runClosure(final Closure done, final Status status) {
+        if (done != null) {
+            done.run(status);
+        }
+    }
+
+    /**
+     * Run closure with OK status in thread pool.
+     */
+    public static Future runClosureInThread(final Closure done) {
+        if (done == null) {
+            return null;
+        }
+        return runClosureInThread(done, Status.OK());
+    }
+
+    /**
+     * Run a task in thread pool,returns the future object.
+     */
+    public static Future runInThread(final Runnable runnable) {
+        return CLOSURE_EXECUTOR.submit(runnable);
+    }
+
+    /**
+     * Run closure with status in thread pool.
+     */
+    @SuppressWarnings("Convert2Lambda")
+    public static Future runClosureInThread(final Closure done, final Status status) {
+        if (done == null) {
+            return null;
+        }
+
+        return runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                try {
+                    done.run(status);
+                } catch (final Throwable t) {
+                    LOG.error("Fail to run done closure", t);
+                }
+            }
+        });
+    }
+
+    /**
+     * Close a closeable.
+     */
+    public static int closeQuietly(final Closeable closeable) {
+        if (closeable == null) {
+            return 0;
+        }
+        try {
+            closeable.close();
+            return 0;
+        } catch (final IOException e) {
+            LOG.error("Fail to close", e);
+            return RaftError.EIO.getNumber();
+        }
+    }
+
+    /**
+     * Get system CPUs count.
+     */
+    public static int cpus() {
+        return CPUS;
+    }
+
+    /**
+     * Get java process id.
+     */
+    public static long getProcessId(final long fallback) {
+        // Note: may fail in some JVM implementations
+        // therefore fallback has to be provided
+
+        // something like '@', at least in SUN / Oracle JVMs
+        final String jvmName = ManagementFactory.getRuntimeMXBean().getName();
+        final int index = jvmName.indexOf('@');
+
+        if (index < 1) {
+            // part before '@' empty (index = 0) / '@' not found (index = -1)
+            return fallback;
+        }
+
+        try {
+            return Long.parseLong(jvmName.substring(0, index));
+        } catch (final NumberFormatException e) {
+            // ignore
+        }
+        return fallback;
+    }
+
+    /**
+     * Default init and expand buffer size, it can be set by -Djraft.byte_buf.size=n, default 1024.
+     */
+    public static final int RAFT_DATA_BUF_SIZE            = SystemPropertyUtil.getInt("jraft.byte_buf.size", 1024);
+
+    /**
+     * Default max {@link ByteBufferCollector} size per thread for recycle, it can be set by
+     * -Djraft.max_collector_size_per_thread, default 256
+     */
+    public static final int MAX_COLLECTOR_SIZE_PER_THREAD = SystemPropertyUtil.getInt(
+                                                              "jraft.max_collector_size_per_thread", 256);
+
+    /**
+     * Expand byte buffer for 1024 bytes.
+     */
+    public static ByteBuffer expandByteBuffer(final ByteBuffer buf) {
+        return expandByteBufferAtLeast(buf, RAFT_DATA_BUF_SIZE);
+    }
+
+    /**
+     * Allocate a byte buffer with size.
+     */
+    public static ByteBuffer allocate(final int size) {
+        return ByteBuffer.allocate(size);
+    }
+
+    /**
+     * Allocate a byte buffer with {@link #RAFT_DATA_BUF_SIZE}
+     */
+    public static ByteBuffer allocate() {
+        return allocate(RAFT_DATA_BUF_SIZE);
+    }
+
+    /**
+     * Expand byte buffer at least minLength.
+     */
+    public static ByteBuffer expandByteBufferAtLeast(final ByteBuffer buf, final int minLength) {
+        final int newCapacity = minLength > RAFT_DATA_BUF_SIZE ? minLength : RAFT_DATA_BUF_SIZE;
+        final ByteBuffer newBuf = ByteBuffer.allocate(buf.capacity() + newCapacity);
+        buf.flip();
+        newBuf.put(buf);
+        return newBuf;
+    }
+
+    /**
+     * Expand byte buffer at most maxLength.
+     */
+    public static ByteBuffer expandByteBufferAtMost(final ByteBuffer buf, final int maxLength) {
+        final int newCapacity = maxLength > RAFT_DATA_BUF_SIZE || maxLength <= 0 ? RAFT_DATA_BUF_SIZE : maxLength;
+        final ByteBuffer newBuf = ByteBuffer.allocate(buf.capacity() + newCapacity);
+        buf.flip();
+        newBuf.put(buf);
+        return newBuf;
+    }
+
+    /**
+     * ANY IP address 0.0.0.0
+     */
+    public static final String IP_ANY = "0.0.0.0";
+
+    /**
+     * Gets the current monotonic time in milliseconds.
+     */
+    public static long monotonicMs() {
+        return TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
+    }
+
+    /**
+     * Returns the current time in milliseconds, it's not monotonic, would be forwarded/backward by
+     * clock synchronous.
+     */
+    public static long nowMs() {
+        return System.currentTimeMillis();
+    }
+
+    /**
+     * Gets the current monotonic time in microseconds.
+     */
+    public static long monotonicUs() {
+        return TimeUnit.NANOSECONDS.toMicros(System.nanoTime());
+    }
+
+    /**
+     * Get string bytes in UTF-8 charset.
+     */
+    public static byte[] getBytes(final String s) {
+        return s.getBytes(StandardCharsets.UTF_8);
+    }
+
+    public static  T withLockObject(final T obj) {
+        return Requires.requireNonNull(obj, "obj");
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    public static boolean atomicMoveFile(final File source, final File target, final boolean sync) throws IOException {
+        // Move temp file to target path atomically.
+        // The code comes from
+        // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/util/AtomicFileWriter.java#L187
+        Requires.requireNonNull(source, "source");
+        Requires.requireNonNull(target, "target");
+        final Path sourcePath = source.toPath();
+        final Path targetPath = target.toPath();
+        boolean success;
+        try {
+            success = Files.move(sourcePath, targetPath, StandardCopyOption.ATOMIC_MOVE) != null;
+        } catch (final IOException e) {
+            // If it falls here that can mean many things. Either that the atomic move is not supported,
+            // or something wrong happened. Anyway, let's try to be over-diagnosing
+            if (e instanceof AtomicMoveNotSupportedException) {
+                LOG.warn("Atomic move not supported, falling back to non-atomic move, error: {}.", e.getMessage());
+            } else {
+                LOG.warn("Unable to move atomically, falling back to non-atomic move, error: {}.", e.getMessage());
+            }
+
+            if (target.exists()) {
+                LOG.info("The target file {} was already existing.", targetPath);
+            }
+
+            try {
+                success = Files.move(sourcePath, targetPath, StandardCopyOption.REPLACE_EXISTING) != null;
+            } catch (final IOException e1) {
+                e1.addSuppressed(e);
+                LOG.warn("Unable to move {} to {}. Attempting to delete {} and abandoning.", sourcePath, targetPath,
+                    sourcePath);
+                try {
+                    Files.deleteIfExists(sourcePath);
+                } catch (final IOException e2) {
+                    e2.addSuppressed(e1);
+                    LOG.warn("Unable to delete {}, good bye then!", sourcePath);
+                    throw e2;
+                }
+
+                throw e1;
+            }
+        }
+        if (success && sync) {
+            File dir = target.getParentFile();
+            // fsync on target parent dir.
+            fsync(dir);
+        }
+        return success;
+    }
+
+    /**
+     * Calls fsync on a file or directory.
+     * @param file file or directory
+     * @throws IOException if an I/O error occurs
+     */
+    public static void fsync(final File file) throws IOException {
+        final boolean isDir = file.isDirectory();
+        // can't fsync on windows.
+        if (isDir && Platform.isWindows()) {
+            LOG.warn("Unable to fsync directory {} on windows.", file);
+            return;
+        }
+        try (final FileChannel fc = FileChannel.open(file.toPath(), isDir ? StandardOpenOption.READ
+            : StandardOpenOption.WRITE)) {
+            fc.force(true);
+        }
+    }
+
+    /**
+     * Unmap mappedByteBuffer
+     * See https://stackoverflow.com/questions/2972986/how-to-unmap-a-file-from-memory-mapped-using-filechannel-in-java
+     */
+    public static void unmap(final MappedByteBuffer cb) {
+        // JavaSpecVer: 1.6, 1.7, 1.8, 9, 10
+        final boolean isOldJDK = System.getProperty("java.specification.version", "99").startsWith("1.");
+        try {
+            if (isOldJDK) {
+                final Method cleaner = cb.getClass().getMethod("cleaner");
+                cleaner.setAccessible(true);
+                final Method clean = Class.forName("sun.misc.Cleaner").getMethod("clean");
+                clean.setAccessible(true);
+                clean.invoke(cleaner.invoke(cb));
+            } else {
+                Class unsafeClass;
+                try {
+                    unsafeClass = Class.forName("sun.misc.Unsafe");
+                } catch (final Exception ex) {
+                    // jdk.internal.misc.Unsafe doesn't yet have an invokeCleaner() method,
+                    // but that method should be added if sun.misc.Unsafe is removed.
+                    unsafeClass = Class.forName("jdk.internal.misc.Unsafe");
+                }
+                final Method clean = unsafeClass.getMethod("invokeCleaner", ByteBuffer.class);
+                clean.setAccessible(true);
+                final Field theUnsafeField = unsafeClass.getDeclaredField("theUnsafe");
+                theUnsafeField.setAccessible(true);
+                final Object theUnsafe = theUnsafeField.get(null);
+                clean.invoke(theUnsafe, cb);
+            }
+        } catch (final Exception ex) {
+            LOG.error("Fail to un-mapped segment file.", ex);
+        }
+    }
+
+    public static String getString(final byte[] bs, final int off, final int len) {
+        return new String(bs, off, len, StandardCharsets.UTF_8);
+    }
+
+    public static final String IPV6_START_MARK     = "[";
+
+    public static final String IPV6_END_MARK       = "]";
+
+    private static final int   IPV6_ADDRESS_LENGTH = 16;
+
+    /**
+     * check whether the ip address is IPv6.
+     *
+     * @param addr ip address
+     * @return boolean
+     */
+    public static boolean isIPv6(final String addr) {
+        try {
+            return InetAddress.getByName(addr).getAddress().length == IPV6_ADDRESS_LENGTH;
+        } catch (Exception e) {
+            throw new IllegalArgumentException(e);
+        }
+    }
+
+    /**
+     * Parse peerId from string that generated by {@link #toString()}
+     * This method can support parameter string values are below:
+     *
+     * 
+     * PeerId.parse("a:b")          = new PeerId("a", "b", 0 , -1)
+     * PeerId.parse("a:b:c")        = new PeerId("a", "b", "c", -1)
+     * PeerId.parse("a:b::d")       = new PeerId("a", "b", 0, "d")
+     * PeerId.parse("a:b:c:d")      = new PeerId("a", "b", "c", "d")
+     * 

+     *
+     */
+    public static String[] parsePeerId(final String s) {
+        if (s.startsWith(IPV6_START_MARK) && StringUtils.containsIgnoreCase(s, IPV6_END_MARK)) {
+            String ipv6Addr;
+            if (s.endsWith(IPV6_END_MARK)) {
+                ipv6Addr = s;
+            } else {
+                ipv6Addr = s.substring(0, (s.indexOf(IPV6_END_MARK) + 1));
+            }
+            if (!isIPv6(ipv6Addr)) {
+                throw new IllegalArgumentException("The IPv6 address(\"" + ipv6Addr + "\") is incorrect.");
+            }
+            String tempString = s.substring((s.indexOf(ipv6Addr) + ipv6Addr.length()));
+            if (tempString.startsWith(":")) {
+                tempString = tempString.substring(1);
+            }
+            String[] tempArr = StringUtils.splitPreserveAllTokens(tempString, ':');
+            String[] result = new String[1 + tempArr.length];
+            result[0] = ipv6Addr;
+            System.arraycopy(tempArr, 0, result, 1, tempArr.length);
+            return result;
+        } else {
+            return StringUtils.splitPreserveAllTokens(s, ':');
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphore.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphore.java
new file mode 100644
index 0000000..3a2a0bf
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphore.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.io.Serializable;
+import java.util.concurrent.Semaphore;
+
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * An implementation of adjustable semaphore.
+ *
+ * Written by Marshall Pierce and released to the public domain
+ * See: http://blog.teamlazerbeez.com/2009/04/20/javas-semaphore-resizing/
+ */
+public final class AdjustableSemaphore implements Serializable {
+
+    private static final long         serialVersionUID = -266635933115069924L;
+
+    private final ResizeableSemaphore semaphore        = new ResizeableSemaphore(0);
+    private volatile int              maxPermits       = 0;
+
+    public AdjustableSemaphore() {
+    }
+
+    public AdjustableSemaphore(int maxPermits) {
+        Requires.requireTrue(maxPermits >= 0, "maxPermits must be a non-negative value");
+        setMaxPermits(maxPermits);
+    }
+
+    public int getMaxPermits() {
+        return maxPermits;
+    }
+
+    /**
+     * Adjusts the maximum number of available permits.
+     *
+     * @param newMaxPermits max number of permits
+     */
+    public synchronized void setMaxPermits(final int newMaxPermits) {
+        Requires.requireTrue(newMaxPermits >= 0, "Semaphore permits must be at least 0, but was " + newMaxPermits);
+
+        final int delta = newMaxPermits - this.maxPermits;
+
+        if (delta == 0) {
+            return;
+        } else if (delta > 0) {
+            this.semaphore.release(delta);
+        } else {
+            this.semaphore.reducePermits(-delta);
+        }
+
+        this.maxPermits = newMaxPermits;
+    }
+
+    /**
+     * Releases a permit, returning it to the semaphore.
+     */
+    public void release() {
+        this.semaphore.release();
+    }
+
+    /**
+     * Acquires a permit from this semaphore, blocking until one is
+     * available, or the thread is {@linkplain Thread#interrupt interrupted}.
+     *
+     * @throws InterruptedException if the current thread is interrupted
+     */
+    public void acquire() throws InterruptedException {
+        this.semaphore.acquire();
+    }
+
+    /**
+     * Acquires a permit from this semaphore, only if one is available at the
+     * time of invocation.
+     *
+     * @return {@code true} if a permit was acquired and {@code false}
+     * otherwise
+     */
+    public boolean tryAcquire() {
+        return this.semaphore.tryAcquire();
+    }
+
+    /**
+     * Returns the current number of permits available in this semaphore.
+     *
+     * @return the number of permits available in this semaphore
+     */
+    public int availablePermits() {
+        return this.semaphore.availablePermits();
+    }
+
+    /**
+     * Returns if the permits is available of the semaphore.
+     *
+     * @return {@code true} if current number of permits > 0
+     */
+    public boolean isAvailable() {
+        return availablePermits() > 0;
+    }
+
+    private static final class ResizeableSemaphore extends Semaphore {
+
+        private static final long serialVersionUID = 1204115455517785966L;
+
+        public ResizeableSemaphore(int permits) {
+            super(permits);
+        }
+
+        @Override
+        protected void reducePermits(final int reduction) {
+            super.reducePermits(reduction);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ConcurrentHashSet.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ConcurrentHashSet.java
new file mode 100644
index 0000000..1769e8b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ConcurrentHashSet.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.AbstractSet;
+import java.util.Iterator;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * Concurrent hash set.
+ *
+ * Fork from bolt
+ * 
+ * @author yunliang.shi
+ */
+public class ConcurrentHashSet extends AbstractSet {
+    private ConcurrentHashMap map;
+
+    /**
+     * constructor
+     */
+    public ConcurrentHashSet() {
+        super();
+        map = new ConcurrentHashMap<>();
+    }
+
+    /**
+     * return the size of the map
+     * @see java.util.AbstractCollection#size()
+     */
+    @Override
+    public int size() {
+        return map.size();
+    }
+
+    /**
+     * 
+     * @see java.util.AbstractCollection#contains(Object)
+     */
+    @SuppressWarnings("SuspiciousMethodCalls")
+    @Override
+    public boolean contains(Object o) {
+        return map.containsKey(o);
+    }
+
+    /**
+     *
+     * @see java.util.AbstractCollection#iterator()
+     */
+    @Override
+    public Iterator iterator() {
+        return map.keySet().iterator();
+    }
+
+    /**
+     * add an obj to set, if exist, return false, else return true
+     * @see java.util.AbstractCollection#add(Object)
+     */
+    @Override
+    public boolean add(E o) {
+        return map.putIfAbsent(o, Boolean.TRUE) == null;
+    }
+
+    /**
+     *
+     * @see java.util.AbstractCollection#remove(Object)
+     */
+    @Override
+    public boolean remove(Object o) {
+        return map.remove(o) != null;
+    }
+
+    /**
+     * clear the set
+     * @see java.util.AbstractCollection#clear()
+     */
+    @Override
+    public void clear() {
+        map.clear();
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultExecutorChooserFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultExecutorChooserFactory.java
new file mode 100644
index 0000000..34d1ef5
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultExecutorChooserFactory.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alipay.sofa.jraft.util.Ints;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class DefaultExecutorChooserFactory implements ExecutorChooserFactory {
+
+    public static final DefaultExecutorChooserFactory INSTANCE = new DefaultExecutorChooserFactory();
+
+    @Override
+    public ExecutorChooser newChooser(final SingleThreadExecutor[] executors) {
+        if (Ints.isPowerOfTwo(executors.length)) {
+            return new PowerOfTwoExecutorChooser(executors);
+        } else {
+            return new GenericExecutorChooser(executors);
+        }
+    }
+
+    private DefaultExecutorChooserFactory() {
+    }
+
+    private static class PowerOfTwoExecutorChooser extends AbstractExecutorChooser {
+
+        PowerOfTwoExecutorChooser(SingleThreadExecutor[] executors) {
+            super(executors);
+        }
+
+        @Override
+        public SingleThreadExecutor select(final int index) {
+            return this.executors[index & this.executors.length - 1];
+        }
+    }
+
+    private static class GenericExecutorChooser extends AbstractExecutorChooser {
+
+        protected GenericExecutorChooser(SingleThreadExecutor[] executors) {
+            super(executors);
+        }
+
+        @Override
+        public SingleThreadExecutor select(final int index) {
+            return this.executors[Math.abs(index % this.executors.length)];
+        }
+    }
+
+    private static abstract class AbstractExecutorChooser implements ExecutorChooser {
+
+        protected final AtomicInteger          idx = new AtomicInteger();
+        protected final SingleThreadExecutor[] executors;
+
+        protected AbstractExecutorChooser(SingleThreadExecutor[] executors) {
+            this.executors = executors;
+        }
+
+        @Override
+        public SingleThreadExecutor next() {
+            return select(this.idx.getAndIncrement());
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroup.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroup.java
new file mode 100644
index 0000000..99479e7
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroup.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class DefaultFixedThreadsExecutorGroup implements FixedThreadsExecutorGroup {
+
+    private final SingleThreadExecutor[]                 children;
+    private final Set              readonlyChildren;
+    private final ExecutorChooserFactory.ExecutorChooser chooser;
+
+    public DefaultFixedThreadsExecutorGroup(SingleThreadExecutor[] children) {
+        this(children, DefaultExecutorChooserFactory.INSTANCE.newChooser(children));
+    }
+
+    public DefaultFixedThreadsExecutorGroup(SingleThreadExecutor[] children,
+                                            ExecutorChooserFactory.ExecutorChooser chooser) {
+        this.children = children;
+        this.readonlyChildren = toUnmodifiableSet(this.children);
+        this.chooser = chooser;
+    }
+
+    public DefaultFixedThreadsExecutorGroup(ExecutorService[] executors) {
+        this.children = toSingleThreadExecutors(executors);
+        this.readonlyChildren = toUnmodifiableSet(this.children);
+        this.chooser = DefaultExecutorChooserFactory.INSTANCE.newChooser(this.children);
+    }
+
+    public DefaultFixedThreadsExecutorGroup(ExecutorService[] executors, ExecutorChooserFactory.ExecutorChooser chooser) {
+        this.children = toSingleThreadExecutors(executors);
+        this.readonlyChildren = toUnmodifiableSet(this.children);
+        this.chooser = chooser;
+    }
+
+    @Override
+    public SingleThreadExecutor next() {
+        return this.chooser.next();
+    }
+
+    @Override
+    public void execute(final int index, final Runnable task) {
+        this.chooser.select(index).execute(task);
+    }
+
+    @Override
+    public boolean shutdownGracefully() {
+        boolean success = true;
+        for (final SingleThreadExecutor c : this.children) {
+            success = success && c.shutdownGracefully();
+        }
+        return success;
+    }
+
+    @Override
+    public boolean shutdownGracefully(final long timeout, final TimeUnit unit) {
+        boolean success = true;
+        final long timeoutNanos = unit.toNanos(timeout);
+        final long start = System.nanoTime();
+        for (final SingleThreadExecutor c : this.children) {
+            success = success && c.shutdownGracefully(timeout, unit);
+            if (System.nanoTime() - start > timeoutNanos) {
+                success = false;
+                break;
+            }
+        }
+        return success;
+    }
+
+    @Override
+    public Iterator iterator() {
+        return this.readonlyChildren.iterator();
+    }
+
+    private static SingleThreadExecutor[] toSingleThreadExecutors(final ExecutorService[] executors) {
+        final SingleThreadExecutor[] array = new SingleThreadExecutor[executors.length];
+        for (int i = 0; i < executors.length; i++) {
+            if (executors[i] instanceof SingleThreadExecutor) {
+                array[i] = (SingleThreadExecutor) executors[i];
+            } else {
+                array[i] = new DefaultSingleThreadExecutor(executors[i]);
+            }
+        }
+        return array;
+    }
+
+    private static Set toUnmodifiableSet(final SingleThreadExecutor[] children) {
+        final Set tmp = new LinkedHashSet<>();
+        Collections.addAll(tmp, children);
+        return Collections.unmodifiableSet(tmp);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroupFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroupFactory.java
new file mode 100644
index 0000000..da0fdd0
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultFixedThreadsExecutorGroupFactory.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadFactory;
+
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class DefaultFixedThreadsExecutorGroupFactory implements FixedThreadsExecutorGroupFactory {
+
+    public static final DefaultFixedThreadsExecutorGroupFactory INSTANCE = new DefaultFixedThreadsExecutorGroupFactory();
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final int nThreads, final String poolName,
+                                                      final int maxPendingTasksPerThread) {
+        return newExecutorGroup(nThreads, poolName, maxPendingTasksPerThread, false);
+    }
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final int nThreads, final String poolName,
+                                                      final int maxPendingTasksPerThread, final boolean useMpscQueue) {
+        Requires.requireTrue(nThreads > 0, "nThreads must > 0");
+        final boolean mpsc = useMpscQueue && Utils.USE_MPSC_SINGLE_THREAD_EXECUTOR;
+        final SingleThreadExecutor[] children = new SingleThreadExecutor[nThreads];
+        final ThreadFactory threadFactory = mpsc ? new NamedThreadFactory(poolName, true) : null;
+        for (int i = 0; i < nThreads; i++) {
+            if (mpsc) {
+                children[i] = new MpscSingleThreadExecutor(maxPendingTasksPerThread, threadFactory);
+            } else {
+                children[i] = new DefaultSingleThreadExecutor(poolName, maxPendingTasksPerThread);
+            }
+        }
+        return new DefaultFixedThreadsExecutorGroup(children);
+    }
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final SingleThreadExecutor[] children) {
+        return new DefaultFixedThreadsExecutorGroup(children);
+    }
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final SingleThreadExecutor[] children,
+                                                      final ExecutorChooserFactory.ExecutorChooser chooser) {
+        return new DefaultFixedThreadsExecutorGroup(children, chooser);
+    }
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final ExecutorService[] children) {
+        return new DefaultFixedThreadsExecutorGroup(children);
+    }
+
+    @Override
+    public FixedThreadsExecutorGroup newExecutorGroup(final ExecutorService[] children,
+                                                      final ExecutorChooserFactory.ExecutorChooser chooser) {
+        return new DefaultFixedThreadsExecutorGroup(children, chooser);
+    }
+
+    private DefaultFixedThreadsExecutorGroupFactory() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultSingleThreadExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultSingleThreadExecutor.java
new file mode 100644
index 0000000..fe000fd
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/DefaultSingleThreadExecutor.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class DefaultSingleThreadExecutor implements SingleThreadExecutor {
+
+    private final SingleThreadExecutor singleThreadExecutor;
+
+    /**
+     * Anti-gentleman is not against villains, we believe that you are
+     * providing a single-thread executor.
+     *
+     * @param singleThreadExecutorService a {@link ExecutorService} instance
+     */
+    public DefaultSingleThreadExecutor(ExecutorService singleThreadExecutorService) {
+        this.singleThreadExecutor = wrapSingleThreadExecutor(singleThreadExecutorService);
+    }
+
+    public DefaultSingleThreadExecutor(String poolName, int maxPendingTasks) {
+        this.singleThreadExecutor = createSingleThreadExecutor(poolName, maxPendingTasks);
+    }
+
+    @Override
+    public void execute(final Runnable task) {
+        this.singleThreadExecutor.execute(task);
+    }
+
+    @Override
+    public boolean shutdownGracefully() {
+        return this.singleThreadExecutor.shutdownGracefully();
+    }
+
+    @Override
+    public boolean shutdownGracefully(final long timeout, final TimeUnit unit) {
+        return this.singleThreadExecutor.shutdownGracefully(timeout, unit);
+    }
+
+    private static SingleThreadExecutor wrapSingleThreadExecutor(final ExecutorService executor) {
+        if (executor instanceof SingleThreadExecutor) {
+            return (SingleThreadExecutor) executor;
+        } else {
+            return new SingleThreadExecutor() {
+
+                @Override
+                public boolean shutdownGracefully() {
+                    return ExecutorServiceHelper.shutdownAndAwaitTermination(executor);
+                }
+
+                @Override
+                public boolean shutdownGracefully(final long timeout, final TimeUnit unit) {
+                    return ExecutorServiceHelper.shutdownAndAwaitTermination(executor, unit.toMillis(timeout));
+                }
+
+                @Override
+                public void execute(final Runnable command) {
+                    executor.execute(command);
+                }
+            };
+        }
+    }
+
+    private static SingleThreadExecutor createSingleThreadExecutor(final String poolName, final int maxPendingTasks) {
+        final ExecutorService singleThreadPool = ThreadPoolUtil.newBuilder() //
+            .poolName(poolName) //
+            .enableMetric(true) //
+            .coreThreads(1) //
+            .maximumThreads(1) //
+            .keepAliveSeconds(60L) //
+            .workQueue(new LinkedBlockingQueue<>(maxPendingTasks)) //
+            .threadFactory(new NamedThreadFactory(poolName, true)) //
+            .build();
+
+        return new SingleThreadExecutor() {
+
+            @Override
+            public boolean shutdownGracefully() {
+                return ExecutorServiceHelper.shutdownAndAwaitTermination(singleThreadPool);
+            }
+
+            @Override
+            public boolean shutdownGracefully(final long timeout, final TimeUnit unit) {
+                return ExecutorServiceHelper.shutdownAndAwaitTermination(singleThreadPool, unit.toMillis(timeout));
+            }
+
+            @Override
+            public void execute(final Runnable command) {
+                singleThreadPool.execute(command);
+            }
+        };
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ExecutorChooserFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ExecutorChooserFactory.java
new file mode 100644
index 0000000..8058d51
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/ExecutorChooserFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+/**
+ * Factory that creates new {@link ExecutorChooser}s.
+ *
+ * @author jiachun.fjc
+ */
+public interface ExecutorChooserFactory {
+
+    /**
+     * Returns a new {@link ExecutorChooser}.
+     */
+    ExecutorChooser newChooser(final SingleThreadExecutor[] executors);
+
+    interface ExecutorChooser {
+
+        /**
+         * Returns the next {@link SingleThreadExecutor} to use.
+         */
+        SingleThreadExecutor next();
+
+        /**
+         * Returns the chosen {@link SingleThreadExecutor} to use.
+         */
+        SingleThreadExecutor select(final int index);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroup.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroup.java
new file mode 100644
index 0000000..6c182fe
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroup.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface FixedThreadsExecutorGroup extends Iterable {
+
+    /**
+     * Returns one of the {@link SingleThreadExecutor}s managed by this
+     * {@link FixedThreadsExecutorGroup}.
+     */
+    SingleThreadExecutor next();
+
+    /**
+     * Executes the given task at some time in the future.  The task
+     * execute by a specified thread, which is selected by index.
+     *
+     * @param index index for thread chooser
+     * @param task  the runnable task
+     */
+    void execute(final int index, final Runnable task);
+
+    /**
+     * Shortcut method for {@link #shutdownGracefully(long, TimeUnit)} with
+     * sensible default values.
+     *
+     * @return true if success to shutdown
+     */
+    boolean shutdownGracefully();
+
+    /**
+     * Signals all executors that the caller wants them to be shutdown.
+     *
+     * @param timeout the maximum amount of time to wait until the executor
+     *                is shutdown
+     * @param unit    the unit of {@code timeout}
+     * @return true if success to shutdown
+     */
+    boolean shutdownGracefully(final long timeout, final TimeUnit unit);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroupFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroupFactory.java
new file mode 100644
index 0000000..cf3e929
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/FixedThreadsExecutorGroupFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.ExecutorService;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface FixedThreadsExecutorGroupFactory {
+
+    FixedThreadsExecutorGroup newExecutorGroup(final int nThreads, final String poolName,
+                                               final int maxPendingTasksPerThread);
+
+    FixedThreadsExecutorGroup newExecutorGroup(final int nThreads, final String poolName,
+                                               final int maxPendingTasksPerThread, final boolean useMpscQueue);
+
+    FixedThreadsExecutorGroup newExecutorGroup(final SingleThreadExecutor[] children);
+
+    FixedThreadsExecutorGroup newExecutorGroup(final SingleThreadExecutor[] children,
+                                               final ExecutorChooserFactory.ExecutorChooser chooser);
+
+    FixedThreadsExecutorGroup newExecutorGroup(final ExecutorService[] children);
+
+    FixedThreadsExecutorGroup newExecutorGroup(final ExecutorService[] children,
+                                               final ExecutorChooserFactory.ExecutorChooser chooser);
+
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLock.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLock.java
new file mode 100644
index 0000000..fa16a5a
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLock.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * A {@link ReadWriteLock} that can report some info when the lock is held by a thread very long time.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class LongHeldDetectingReadWriteLock implements ReadWriteLock {
+
+    public enum AcquireMode {
+        Read, Write
+    }
+
+    private final Lock rLock;
+    private final Lock wLock;
+
+    public LongHeldDetectingReadWriteLock(long maxBlockingTimeToReport, TimeUnit unit) {
+        this(false, maxBlockingTimeToReport, unit);
+    }
+
+    public LongHeldDetectingReadWriteLock(boolean fair, long maxBlockingTimeToReport, TimeUnit unit) {
+        final RwLock rwLock = new RwLock(fair);
+        final long maxBlockingNanos = unit.toNanos(maxBlockingTimeToReport);
+        if (maxBlockingNanos > 0) {
+            this.rLock = new LongHeldDetectingLock(AcquireMode.Read, rwLock, maxBlockingNanos);
+            this.wLock = new LongHeldDetectingLock(AcquireMode.Write, rwLock, maxBlockingNanos);
+        } else {
+            this.rLock = rwLock.readLock();
+            this.wLock = rwLock.writeLock();
+        }
+    }
+
+    @Override
+    public Lock readLock() {
+        return this.rLock;
+    }
+
+    @Override
+    public Lock writeLock() {
+        return this.wLock;
+    }
+
+    public abstract void report(final AcquireMode acquireMode, final Thread heldThread,
+                                final Collection queuedThreads, final long blockedNanos);
+
+    static class RwLock extends ReentrantReadWriteLock {
+
+        private static final long serialVersionUID = -1783358548846940445L;
+
+        public RwLock() {
+        }
+
+        public RwLock(boolean fair) {
+            super(fair);
+        }
+
+        @Override
+        public Thread getOwner() {
+            return super.getOwner();
+        }
+
+        @Override
+        public Collection getQueuedThreads() {
+            return super.getQueuedThreads();
+        }
+    }
+
+    class LongHeldDetectingLock implements Lock {
+
+        private final AcquireMode mode;
+        private final RwLock      parent;
+        private final Lock        delegate;
+        private final long        maxBlockingNanos;
+
+        LongHeldDetectingLock(AcquireMode mode, RwLock parent, long maxBlockingNanos) {
+            this.mode = mode;
+            this.parent = parent;
+            this.delegate = mode == AcquireMode.Read ? parent.readLock() : parent.writeLock();
+            this.maxBlockingNanos = maxBlockingNanos;
+        }
+
+        @Override
+        public void lock() {
+            final long start = System.nanoTime();
+            final Thread owner = this.parent.getOwner();
+            try {
+                this.delegate.lock();
+            } finally {
+                final long elapsed = System.nanoTime() - start;
+                if (elapsed > this.maxBlockingNanos) {
+                    report(this.mode, owner, this.parent.getQueuedThreads(), elapsed);
+                }
+            }
+        }
+
+        @Override
+        public void lockInterruptibly() throws InterruptedException {
+            final long start = System.nanoTime();
+            final Thread owner = this.parent.getOwner();
+            try {
+                this.delegate.lockInterruptibly();
+            } finally {
+                final long elapsed = System.nanoTime() - start;
+                if (elapsed > this.maxBlockingNanos) {
+                    report(this.mode, owner, this.parent.getQueuedThreads(), elapsed);
+                }
+            }
+        }
+
+        @Override
+        public boolean tryLock() {
+            return this.delegate.tryLock();
+        }
+
+        @Override
+        public boolean tryLock(final long time, final TimeUnit unit) throws InterruptedException {
+            return this.delegate.tryLock(time, unit);
+        }
+
+        @Override
+        public void unlock() {
+            this.delegate.unlock();
+        }
+
+        @Override
+        public Condition newCondition() {
+            return this.delegate.newCondition();
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutor.java
new file mode 100644
index 0000000..8a4b1e0
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutor.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.util.Mpsc;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MpscSingleThreadExecutor implements SingleThreadExecutor {
+
+    private static final Logger                                              LOG                      = LoggerFactory
+                                                                                                          .getLogger(MpscSingleThreadExecutor.class);
+
+    private static final AtomicIntegerFieldUpdater STATE_UPDATER            = AtomicIntegerFieldUpdater
+                                                                                                          .newUpdater(
+                                                                                                              MpscSingleThreadExecutor.class,
+                                                                                                              "state");
+
+    private static final long                                                DEFAULT_SHUTDOWN_TIMEOUT = 15;
+
+    private static final int                                                 ST_NOT_STARTED           = 1;
+    private static final int                                                 ST_STARTED               = 2;
+    private static final int                                                 ST_SHUTDOWN              = 3;
+    private static final int                                                 ST_TERMINATED            = 4;
+
+    private static final Runnable                                            WAKEUP_TASK              = () -> {};
+
+    private final Queue                                            taskQueue;
+    private final Executor                                                   executor;
+    private final RejectedExecutionHandler                                   rejectedExecutionHandler;
+    private final Set                                              shutdownHooks            = new LinkedHashSet<>();
+    private final Semaphore                                                  threadLock               = new Semaphore(0);
+
+    private volatile int                                                     state                    = ST_NOT_STARTED;
+    private volatile Worker                                                  worker;
+
+    public MpscSingleThreadExecutor(int maxPendingTasks, ThreadFactory threadFactory) {
+        this(maxPendingTasks, threadFactory, RejectedExecutionHandlers.reject());
+    }
+
+    public MpscSingleThreadExecutor(int maxPendingTasks, ThreadFactory threadFactory,
+                                    RejectedExecutionHandler rejectedExecutionHandler) {
+        this.taskQueue = newTaskQueue(maxPendingTasks);
+        this.executor = new ThreadPerTaskExecutor(threadFactory);
+        this.rejectedExecutionHandler = rejectedExecutionHandler;
+    }
+
+    @Override
+    public boolean shutdownGracefully() {
+        return shutdownGracefully(DEFAULT_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS);
+    }
+
+    @Override
+    public boolean shutdownGracefully(final long timeout, final TimeUnit unit) {
+        Requires.requireNonNull(unit, "unit");
+        if (isShutdown()) {
+            return awaitTermination(timeout, unit);
+        }
+
+        boolean wakeup;
+        int oldState;
+        for (;;) {
+            if (isShutdown()) {
+                return awaitTermination(timeout, unit);
+            }
+            int newState;
+            wakeup = true;
+            oldState = this.state;
+            switch (oldState) {
+                case ST_NOT_STARTED:
+                case ST_STARTED:
+                    newState = ST_SHUTDOWN;
+                    break;
+                default:
+                    newState = oldState;
+                    wakeup = false;
+            }
+            if (STATE_UPDATER.compareAndSet(this, oldState, newState)) {
+                break;
+            }
+        }
+
+        if (oldState == ST_NOT_STARTED) {
+            try {
+                doStartWorker();
+            } catch (final Throwable t) {
+                this.state = ST_TERMINATED;
+
+                if (!(t instanceof Exception)) {
+                    // Also rethrow as it may be an OOME for example
+                    throw new RuntimeException(t);
+                }
+                return true;
+            }
+        }
+
+        if (wakeup) {
+            wakeupAndStopWorker();
+        }
+
+        return awaitTermination(timeout, unit);
+    }
+
+    @Override
+    public void execute(final Runnable task) {
+        Requires.requireNonNull(task, "task");
+
+        addTask(task);
+        startWorker();
+        wakeupForTask();
+    }
+
+    /**
+     * Add a {@link Runnable} which will be executed on shutdown of this instance.
+     */
+    public void addShutdownHook(final Runnable task) {
+        execute(() -> MpscSingleThreadExecutor.this.shutdownHooks.add(task));
+    }
+
+    /**
+     * Remove a previous added {@link Runnable} as a shutdown hook.
+     */
+    public void removeShutdownHook(final Runnable task) {
+        execute(() -> MpscSingleThreadExecutor.this.shutdownHooks.remove(task));
+    }
+
+    private boolean runShutdownHooks() {
+        boolean ran = false;
+        // Note shutdown hooks can add / remove shutdown hooks.
+        while (!this.shutdownHooks.isEmpty()) {
+            final List copy = new ArrayList<>(this.shutdownHooks);
+            this.shutdownHooks.clear();
+            for (final Runnable task : copy) {
+                try {
+                    task.run();
+                } catch (final Throwable t) {
+                    LOG.warn("Shutdown hook raised an exception.", t);
+                } finally {
+                    ran = true;
+                }
+            }
+        }
+        return ran;
+    }
+
+    public boolean isShutdown() {
+        return this.state >= ST_SHUTDOWN;
+    }
+
+    public boolean isTerminated() {
+        return this.state == ST_TERMINATED;
+    }
+
+    public boolean inWorkerThread(final Thread thread) {
+        final Worker worker = this.worker;
+        return worker != null && worker.thread == thread;
+    }
+
+    public boolean awaitTermination(final long timeout, final TimeUnit unit) {
+        Requires.requireNonNull(unit, "unit");
+
+        try {
+            if (this.threadLock.tryAcquire(timeout, unit)) {
+                this.threadLock.release();
+            }
+        } catch (final InterruptedException ignored) {
+            // ignored
+        }
+
+        return isTerminated();
+    }
+
+    protected Queue newTaskQueue(final int maxPendingTasks) {
+        return maxPendingTasks == Integer.MAX_VALUE ? Mpsc.newMpscQueue() : Mpsc.newMpscQueue(maxPendingTasks);
+    }
+
+    /**
+     * Add a task to the task queue, or throws a {@link RejectedExecutionException} if
+     * this instance was shutdown before.
+     */
+    protected void addTask(final Runnable task) {
+        if (!offerTask(task)) {
+            reject(task);
+        }
+    }
+
+    protected final boolean offerTask(final Runnable task) {
+        if (isShutdown()) {
+            reject();
+        }
+        return this.taskQueue.offer(task);
+    }
+
+    private void wakeupForTask() {
+        final Worker worker = this.worker;
+        if (worker != null) {
+            worker.notifyIfNeeded();
+        }
+    }
+
+    private void wakeupAndStopWorker() {
+        // Maybe the worker has not initialized yet and cant't be notify, so we
+        // add a wakeup_task first, it may prevent the worker be blocked.
+        this.taskQueue.offer(WAKEUP_TASK);
+        final Worker worker = this.worker;
+        if (worker != null) {
+            worker.notifyAndStop();
+        }
+    }
+
+    private void startWorker() {
+        if (this.state != ST_NOT_STARTED) {
+            // avoid CAS if not needed
+            return;
+        }
+        if (STATE_UPDATER.compareAndSet(this, ST_NOT_STARTED, ST_STARTED)) {
+            try {
+                doStartWorker();
+            } catch (final Throwable t) {
+                this.state = ST_NOT_STARTED;
+                throw new RuntimeException("Fail to start executor", t);
+            }
+        }
+    }
+
+    private void doStartWorker() {
+        this.executor.execute(() -> {
+            MpscSingleThreadExecutor.this.worker = new Worker(Thread.currentThread());
+
+            try {
+                MpscSingleThreadExecutor.this.worker.run();
+            } catch (final Throwable t) {
+                LOG.warn("Unexpected exception from executor: ", t);
+            } finally {
+                for (;;) {
+                    int oldState = MpscSingleThreadExecutor.this.state;
+                    if (oldState >= ST_SHUTDOWN || STATE_UPDATER.compareAndSet(MpscSingleThreadExecutor.this, oldState, ST_SHUTDOWN)) {
+                        break;
+                    }
+                }
+
+                runShutdownHooks();
+
+                MpscSingleThreadExecutor.this.state = ST_TERMINATED;
+                MpscSingleThreadExecutor.this.threadLock.release();
+            }
+        });
+    }
+
+    /**
+     * Offers the task to the associated {@link RejectedExecutionHandler}.
+     *
+     * @param task to reject.
+     */
+    protected final void reject(final Runnable task) {
+        this.rejectedExecutionHandler.rejected(task, this);
+    }
+
+    protected static void reject() {
+        throw new RejectedExecutionException("Executor terminated");
+    }
+
+    private static final AtomicIntegerFieldUpdater NOTIFY_UPDATER = AtomicIntegerFieldUpdater.newUpdater(
+                                                                              Worker.class, "notifyNeeded");
+    private static final int                               NOT_NEEDED     = 0;
+    private static final int                               NEEDED         = 1;
+
+    private class Worker implements Runnable {
+
+        final Thread thread;
+        volatile int notifyNeeded = NOT_NEEDED;
+        boolean      stop         = false;
+
+        private Worker(Thread thread) {
+            this.thread = thread;
+        }
+
+        @Override
+        public void run() {
+            for (;;) {
+                final Runnable task = pollTask();
+                if (task == null) {
+                    // wait task
+                    synchronized (this) {
+                        if (this.stop) {
+                            break;
+                        }
+                        this.notifyNeeded = NEEDED;
+                        try {
+                            // Maybe the outer layer calls shutdown when the worker has not initialized yet,
+                            // so we only wait a little while to recheck the conditions.
+                            wait(1000, 10);
+
+                            if (this.stop || isShutdown()) {
+                                break;
+                            }
+                        } catch (final InterruptedException ignored) {
+                            // ignored
+                        }
+                    }
+                    continue;
+                }
+
+                runTask(task);
+
+                if (isShutdown()) {
+                    break;
+                }
+            }
+
+            runAllTasks();
+        }
+
+        private Runnable pollTask() {
+            return MpscSingleThreadExecutor.this.taskQueue.poll();
+        }
+
+        private void runTask(final Runnable task) {
+            try {
+                task.run();
+            } catch (final Throwable t) {
+                LOG.warn("Caught an unknown error while executing a task", t);
+            }
+        }
+
+        private void runAllTasks() {
+            Runnable task;
+            while ((task = pollTask()) != null) {
+                runTask(task);
+            }
+        }
+
+        private boolean isShuttingDown() {
+            return MpscSingleThreadExecutor.this.state != ST_STARTED;
+        }
+
+        private void notifyIfNeeded() {
+            if (this.notifyNeeded == NOT_NEEDED) {
+                return;
+            }
+            if (NOTIFY_UPDATER.getAndSet(this, NOT_NEEDED) == NEEDED) {
+                synchronized (this) {
+                    notifyAll();
+                }
+            }
+        }
+
+        private void notifyAndStop() {
+            synchronized (this) {
+                this.stop = true;
+                notifyAll();
+            }
+        }
+    }
+
+    private static class ThreadPerTaskExecutor implements Executor {
+
+        private final ThreadFactory threadFactory;
+
+        ThreadPerTaskExecutor(ThreadFactory threadFactory) {
+            this.threadFactory = threadFactory;
+        }
+
+        @Override
+        public void execute(final Runnable task) {
+            this.threadFactory.newThread(task).start();
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandler.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandler.java
new file mode 100644
index 0000000..87c9476
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandler.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+/**
+ * Similar to {@link java.util.concurrent.RejectedExecutionHandler} but specific to {@link SingleThreadExecutor}.
+ *
+ * Reference from netty project.
+ *
+ * @author jiachun.fjc
+ */
+public interface RejectedExecutionHandler {
+
+    /**
+     * Called when someone tried to add a task to {@link SingleThreadExecutor} but
+     * this failed due capacity restrictions.
+     */
+    void rejected(final Runnable task, final SingleThreadExecutor executor);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandlers.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandlers.java
new file mode 100644
index 0000000..ea21988
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/RejectedExecutionHandlers.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.RejectedExecutionException;
+
+
+/**
+ * Expose helper methods which create different {@link RejectedExecutionHandler}s.
+ *
+ * Reference from netty project.
+ *
+ * @author jiachun.fjc
+ */
+public final class RejectedExecutionHandlers {
+
+    private static final RejectedExecutionHandler REJECT = (task, executor) -> {
+        throw new RejectedExecutionException();
+    };
+
+    /**
+     * Returns a {@link RejectedExecutionHandler} that will always just throw
+     * a {@link RejectedExecutionException}.
+     */
+    public static RejectedExecutionHandler reject() {
+        return REJECT;
+    }
+
+    private RejectedExecutionHandlers() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutor.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutor.java
new file mode 100644
index 0000000..fc22fb1
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutor.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface SingleThreadExecutor extends Executor {
+
+    /**
+     * Shortcut method for {@link #shutdownGracefully(long, TimeUnit)} with
+     * sensible default values.
+     * @return true if success to shutdown
+     */
+    boolean shutdownGracefully();
+
+    /**
+     * Signals this executor that the caller wants it to be shutdown.
+     *
+     * @param timeout the maximum amount of time to wait until the executor
+     *                is shutdown
+     * @param unit    the unit of {@code timeout}
+     * @return true if success to shutdown
+     */
+    boolean shutdownGracefully(final long timeout, final TimeUnit unit);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/IntegerFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/IntegerFieldUpdater.java
new file mode 100644
index 0000000..658b635
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/IntegerFieldUpdater.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface IntegerFieldUpdater {
+
+    void set(final U obj, final int newValue);
+
+    int get(final U obj);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/LongFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/LongFieldUpdater.java
new file mode 100644
index 0000000..6c0b04b
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/LongFieldUpdater.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface LongFieldUpdater {
+
+    void set(final U obj, final long newValue);
+
+    long get(final U obj);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReferenceFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReferenceFieldUpdater.java
new file mode 100644
index 0000000..a15a7f0
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReferenceFieldUpdater.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface ReferenceFieldUpdater {
+
+    void set(final U obj, final W newValue);
+
+    W get(final U obj);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionIntegerFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionIntegerFieldUpdater.java
new file mode 100644
index 0000000..442ca1c
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionIntegerFieldUpdater.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+final class ReflectionIntegerFieldUpdater implements IntegerFieldUpdater {
+
+    private final Field field;
+
+    ReflectionIntegerFieldUpdater(Class tClass, String fieldName) throws NoSuchFieldException {
+        this.field = tClass.getDeclaredField(fieldName);
+        this.field.setAccessible(true);
+    }
+
+    @Override
+    public void set(final U obj, final int newValue) {
+        try {
+            this.field.set(obj, newValue);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public int get(final U obj) {
+        try {
+            return (Integer) this.field.get(obj);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionLongFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionLongFieldUpdater.java
new file mode 100644
index 0000000..eecb5ce
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionLongFieldUpdater.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+final class ReflectionLongFieldUpdater implements LongFieldUpdater {
+
+    private final Field field;
+
+    ReflectionLongFieldUpdater(Class tClass, String fieldName) throws NoSuchFieldException {
+        this.field = tClass.getDeclaredField(fieldName);
+        this.field.setAccessible(true);
+    }
+
+    @Override
+    public void set(final U obj, final long newValue) {
+        try {
+            this.field.set(obj, newValue);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public long get(final U obj) {
+        try {
+            return (Long) this.field.get(obj);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionReferenceFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionReferenceFieldUpdater.java
new file mode 100644
index 0000000..a23d00f
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ReflectionReferenceFieldUpdater.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("unchecked")
+final class ReflectionReferenceFieldUpdater implements ReferenceFieldUpdater {
+
+    private final Field field;
+
+    ReflectionReferenceFieldUpdater(Class tClass, String fieldName) throws NoSuchFieldException {
+        this.field = tClass.getDeclaredField(fieldName);
+        this.field.setAccessible(true);
+    }
+
+    @Override
+    public void set(final U obj, final W newValue) {
+        try {
+            this.field.set(obj, newValue);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public W get(final U obj) {
+        try {
+            return (W) this.field.get(obj);
+        } catch (final IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ThrowUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ThrowUtil.java
new file mode 100644
index 0000000..c179a89
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/ThrowUtil.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+/**
+ * Throwing tool.
+ *
+ * @author jiachun.fjc
+ */
+public final class ThrowUtil {
+
+    private static final ReferenceFieldUpdater causeUpdater = Updaters.newReferenceFieldUpdater(
+                                                                                      Throwable.class, "cause");
+
+    /**
+     * Raises an exception bypassing compiler checks for checked exceptions.
+     */
+    public static void throwException(final Throwable t) {
+        if (UnsafeUtil.hasUnsafe()) {
+            UnsafeUtil.throwException(t);
+        } else {
+            ThrowUtil.throwException0(t);
+        }
+    }
+
+    /**
+     * private static  void throwException0(java.lang.Throwable) throws E;
+     *      flags: ACC_PRIVATE, ACC_STATIC
+     *      Code:
+     *      stack=1, locals=1, args_size=1
+     *          0: aload_0
+     *          1: athrow
+     *      ...
+     *  Exceptions:
+     *      throws java.lang.Throwable
+     */
+    @SuppressWarnings("unchecked")
+    private static  void throwException0(final Throwable t) throws E {
+        throw (E) t;
+    }
+
+    public static  T cutCause(final T cause) {
+        Throwable rootCause = cause;
+        while (rootCause.getCause() != null) {
+            rootCause = rootCause.getCause();
+        }
+
+        if (rootCause != cause) {
+            cause.setStackTrace(rootCause.getStackTrace());
+            causeUpdater.set(cause, cause);
+        }
+        return cause;
+    }
+
+    private ThrowUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeIntegerFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeIntegerFieldUpdater.java
new file mode 100644
index 0000000..4bc081c
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeIntegerFieldUpdater.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+import sun.misc.Unsafe;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+final class UnsafeIntegerFieldUpdater implements IntegerFieldUpdater {
+
+    private final long   offset;
+    private final Unsafe unsafe;
+
+    UnsafeIntegerFieldUpdater(Unsafe unsafe, Class tClass, String fieldName) throws NoSuchFieldException {
+        final Field field = tClass.getDeclaredField(fieldName);
+        if (unsafe == null) {
+            throw new NullPointerException("unsafe");
+        }
+        this.unsafe = unsafe;
+        this.offset = unsafe.objectFieldOffset(field);
+    }
+
+    @Override
+    public void set(final U obj, final int newValue) {
+        this.unsafe.putInt(obj, this.offset, newValue);
+    }
+
+    @Override
+    public int get(final U obj) {
+        return this.unsafe.getInt(obj, this.offset);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeLongFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeLongFieldUpdater.java
new file mode 100644
index 0000000..d3f5705
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeLongFieldUpdater.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+import sun.misc.Unsafe;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+final class UnsafeLongFieldUpdater implements LongFieldUpdater {
+
+    private final long   offset;
+    private final Unsafe unsafe;
+
+    UnsafeLongFieldUpdater(Unsafe unsafe, Class tClass, String fieldName) throws NoSuchFieldException {
+        final Field field = tClass.getDeclaredField(fieldName);
+        if (unsafe == null) {
+            throw new NullPointerException("unsafe");
+        }
+        this.unsafe = unsafe;
+        this.offset = unsafe.objectFieldOffset(field);
+    }
+
+    @Override
+    public void set(final U obj, final long newValue) {
+        this.unsafe.putLong(obj, this.offset, newValue);
+    }
+
+    @Override
+    public long get(final U obj) {
+        return this.unsafe.getLong(obj, this.offset);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeReferenceFieldUpdater.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeReferenceFieldUpdater.java
new file mode 100644
index 0000000..9c32c56
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeReferenceFieldUpdater.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+import sun.misc.Unsafe;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("unchecked")
+final class UnsafeReferenceFieldUpdater implements ReferenceFieldUpdater {
+
+    private final long   offset;
+    private final Unsafe unsafe;
+
+    UnsafeReferenceFieldUpdater(Unsafe unsafe, Class tClass, String fieldName) throws NoSuchFieldException {
+        final Field field = tClass.getDeclaredField(fieldName);
+        if (unsafe == null) {
+            throw new NullPointerException("unsafe");
+        }
+        this.unsafe = unsafe;
+        this.offset = unsafe.objectFieldOffset(field);
+    }
+
+    @Override
+    public void set(final U obj, final W newValue) {
+        this.unsafe.putObject(obj, this.offset, newValue);
+    }
+
+    @Override
+    public W get(final U obj) {
+        return (W) this.unsafe.getObject(obj, this.offset);
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtf8Util.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtf8Util.java
new file mode 100644
index 0000000..d3706ea
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtf8Util.java
@@ -0,0 +1,481 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.alipay.sofa.jraft.util.internal;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import static java.lang.Character.MAX_SURROGATE;
+import static java.lang.Character.MIN_HIGH_SURROGATE;
+import static java.lang.Character.MIN_LOW_SURROGATE;
+import static java.lang.Character.MIN_SUPPLEMENTARY_CODE_POINT;
+import static java.lang.Character.MIN_SURROGATE;
+import static java.lang.Character.isSurrogatePair;
+import static java.lang.Character.toCodePoint;
+
+/**
+ *
+ * Refer to the implementation of protobuf: https://github.com/protocolbuffers/protobuf/blob/master/java/core/src/main/java/com/google/protobuf/Utf8.java.
+ */
+public final class UnsafeUtf8Util {
+
+    /**
+     * Maximum number of bytes per Java UTF-16 char in UTF-8.
+     *
+     * @see java.nio.charset.CharsetEncoder#maxBytesPerChar()
+     */
+    public static final int MAX_BYTES_PER_CHAR = 3;
+
+    public static String decodeUtf8(byte[] bytes, int index, int size) {
+        if ((index | size | bytes.length - index - size) < 0) {
+            throw new ArrayIndexOutOfBoundsException("buffer length=" + bytes.length + ", index=" + index + ", size="
+                                                     + size);
+        }
+
+        int offset = index;
+        final int limit = offset + size;
+
+        // The longest possible resulting String is the same as the number of input bytes, when it is
+        // all ASCII. For other cases, this over-allocates and we will truncate in the end.
+        char[] resultArr = new char[size];
+        int resultPos = 0;
+
+        // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
+        // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
+        while (offset < limit) {
+            byte b = UnsafeUtil.getByte(bytes, offset);
+            if (!DecodeUtil.isOneByte(b)) {
+                break;
+            }
+            offset++;
+            DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+        }
+
+        while (offset < limit) {
+            byte byte1 = UnsafeUtil.getByte(bytes, offset++);
+            if (DecodeUtil.isOneByte(byte1)) {
+                DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
+                // It's common for there to be multiple ASCII characters in a run mixed in, so add an
+                // extra optimized loop to take care of these runs.
+                while (offset < limit) {
+                    byte b = UnsafeUtil.getByte(bytes, offset);
+                    if (!DecodeUtil.isOneByte(b)) {
+                        break;
+                    }
+                    offset++;
+                    DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+                }
+            } else if (DecodeUtil.isTwoBytes(byte1)) {
+                if (offset >= limit) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleTwoBytes(byte1, /* byte2 */UnsafeUtil.getByte(bytes, offset++), resultArr,
+                    resultPos++);
+            } else if (DecodeUtil.isThreeBytes(byte1)) {
+                if (offset >= limit - 1) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleThreeBytes(byte1,
+                /* byte2 */UnsafeUtil.getByte(bytes, offset++),
+                /* byte3 */UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++);
+            } else {
+                if (offset >= limit - 2) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleFourBytes(byte1,
+                /* byte2 */UnsafeUtil.getByte(bytes, offset++),
+                /* byte3 */UnsafeUtil.getByte(bytes, offset++),
+                /* byte4 */UnsafeUtil.getByte(bytes, offset++), resultArr, resultPos++);
+                // 4-byte case requires two chars.
+                resultPos++;
+            }
+        }
+
+        if (resultPos < resultArr.length) {
+            resultArr = Arrays.copyOf(resultArr, resultPos);
+        }
+        return UnsafeUtil.moveToString(resultArr);
+    }
+
+    public static String decodeUtf8Direct(ByteBuffer buffer, int index, int size) {
+        // Bitwise OR combines the sign bits so any negative value fails the check.
+        if ((index | size | buffer.limit() - index - size) < 0) {
+            throw new ArrayIndexOutOfBoundsException("buffer limit=" + buffer.limit() + ", index=" + index + ", limit="
+                                                     + size);
+        }
+        long address = UnsafeUtil.addressOffset(buffer) + index;
+        final long addressLimit = address + size;
+
+        // The longest possible resulting String is the same as the number of input bytes, when it is
+        // all ASCII. For other cases, this over-allocates and we will truncate in the end.
+        char[] resultArr = new char[size];
+        int resultPos = 0;
+
+        // Optimize for 100% ASCII (Hotspot loves small simple top-level loops like this).
+        // This simple loop stops when we encounter a byte >= 0x80 (i.e. non-ASCII).
+        while (address < addressLimit) {
+            byte b = UnsafeUtil.getByte(address);
+            if (!DecodeUtil.isOneByte(b)) {
+                break;
+            }
+            address++;
+            DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+        }
+
+        while (address < addressLimit) {
+            byte byte1 = UnsafeUtil.getByte(address++);
+            if (DecodeUtil.isOneByte(byte1)) {
+                DecodeUtil.handleOneByte(byte1, resultArr, resultPos++);
+                // It's common for there to be multiple ASCII characters in a run mixed in, so add an
+                // extra optimized loop to take care of these runs.
+                while (address < addressLimit) {
+                    byte b = UnsafeUtil.getByte(address);
+                    if (!DecodeUtil.isOneByte(b)) {
+                        break;
+                    }
+                    address++;
+                    DecodeUtil.handleOneByte(b, resultArr, resultPos++);
+                }
+            } else if (DecodeUtil.isTwoBytes(byte1)) {
+                if (address >= addressLimit) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleTwoBytes(byte1, /* byte2 */UnsafeUtil.getByte(address++), resultArr, resultPos++);
+            } else if (DecodeUtil.isThreeBytes(byte1)) {
+                if (address >= addressLimit - 1) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleThreeBytes(byte1,
+                /* byte2 */UnsafeUtil.getByte(address++),
+                /* byte3 */UnsafeUtil.getByte(address++), resultArr, resultPos++);
+            } else {
+                if (address >= addressLimit - 2) {
+                    throw invalidUtf8();
+                }
+                DecodeUtil.handleFourBytes(byte1,
+                /* byte2 */UnsafeUtil.getByte(address++),
+                /* byte3 */UnsafeUtil.getByte(address++),
+                /* byte4 */UnsafeUtil.getByte(address++), resultArr, resultPos++);
+                // 4-byte case requires two chars.
+                resultPos++;
+            }
+        }
+
+        if (resultPos < resultArr.length) {
+            resultArr = Arrays.copyOf(resultArr, resultPos);
+        }
+        return UnsafeUtil.moveToString(resultArr);
+    }
+
+    public static int encodeUtf8(CharSequence in, byte[] out, int offset, int length) {
+        long outIx = offset;
+        final long outLimit = outIx + length;
+        final int inLimit = in.length();
+        if (inLimit > length || out.length - length < offset) {
+            // Not even enough room for an ASCII-encoded string.
+            throw new ArrayIndexOutOfBoundsException("Failed writing " + in.charAt(inLimit - 1) + " at index "
+                                                     + (offset + length));
+        }
+
+        // Designed to take advantage of
+        // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
+        int inIx = 0;
+        for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) {
+            UnsafeUtil.putByte(out, outIx++, (byte) c);
+        }
+        if (inIx == inLimit) {
+            // We're done, it was ASCII encoded.
+            return (int) outIx;
+        }
+
+        for (char c; inIx < inLimit; ++inIx) {
+            c = in.charAt(inIx);
+            if (c < 0x80 && outIx < outLimit) {
+                UnsafeUtil.putByte(out, outIx++, (byte) c);
+            } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes
+                UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 6) | (c >>> 6)));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c)));
+            } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) {
+                // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes
+                UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 5) | (c >>> 12)));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (c >>> 6))));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & c)));
+            } else if (outIx <= outLimit - 4L) {
+                // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8
+                // bytes
+                final char low;
+                if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) {
+                    throw new IllegalArgumentException("Unpaired surrogate at index " + (inIx - 1) + " of " + inLimit);
+                }
+                int codePoint = toCodePoint(c, low);
+                UnsafeUtil.putByte(out, outIx++, (byte) ((0xF << 4) | (codePoint >>> 18)));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12))));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6))));
+                UnsafeUtil.putByte(out, outIx++, (byte) (0x80 | (0x3F & codePoint)));
+            } else {
+                if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE)
+                    && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) {
+                    // We are surrogates and we're not a surrogate pair.
+                    throw new IllegalArgumentException("Unpaired surrogate at index " + inIx + " of " + inLimit);
+                }
+                // Not enough space in the output buffer.
+                throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx);
+            }
+        }
+
+        // All bytes have been encoded.
+        return (int) outIx;
+    }
+
+    public static void encodeUtf8Direct(CharSequence in, ByteBuffer out) {
+        final long address = UnsafeUtil.addressOffset(out);
+        long outIx = address + out.position();
+        final long outLimit = address + out.limit();
+        final int inLimit = in.length();
+        if (inLimit > outLimit - outIx) {
+            // Not even enough room for an ASCII-encoded string.
+            throw new ArrayIndexOutOfBoundsException("Failed writing " + in.charAt(inLimit - 1) + " at index "
+                                                     + out.limit());
+        }
+
+        // Designed to take advantage of
+        // https://wikis.oracle.com/display/HotSpotInternals/RangeCheckElimination
+        int inIx = 0;
+        for (char c; inIx < inLimit && (c = in.charAt(inIx)) < 0x80; ++inIx) {
+            UnsafeUtil.putByte(outIx++, (byte) c);
+        }
+        if (inIx == inLimit) {
+            // We're done, it was ASCII encoded.
+            out.position((int) (outIx - address));
+            return;
+        }
+
+        for (char c; inIx < inLimit; ++inIx) {
+            c = in.charAt(inIx);
+            if (c < 0x80 && outIx < outLimit) {
+                UnsafeUtil.putByte(outIx++, (byte) c);
+            } else if (c < 0x800 && outIx <= outLimit - 2L) { // 11 bits, two UTF-8 bytes
+                UnsafeUtil.putByte(outIx++, (byte) ((0xF << 6) | (c >>> 6)));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & c)));
+            } else if ((c < MIN_SURROGATE || MAX_SURROGATE < c) && outIx <= outLimit - 3L) {
+                // Maximum single-char code point is 0xFFFF, 16 bits, three UTF-8 bytes
+                UnsafeUtil.putByte(outIx++, (byte) ((0xF << 5) | (c >>> 12)));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (c >>> 6))));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & c)));
+            } else if (outIx <= outLimit - 4L) {
+                // Minimum code point represented by a surrogate pair is 0x10000, 17 bits, four UTF-8
+                // bytes
+                final char low;
+                if (inIx + 1 == inLimit || !isSurrogatePair(c, (low = in.charAt(++inIx)))) {
+                    throw new IllegalArgumentException("Unpaired surrogate at index " + (inIx - 1) + " of " + inLimit);
+                }
+                int codePoint = toCodePoint(c, low);
+                UnsafeUtil.putByte(outIx++, (byte) ((0xF << 4) | (codePoint >>> 18)));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 12))));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & (codePoint >>> 6))));
+                UnsafeUtil.putByte(outIx++, (byte) (0x80 | (0x3F & codePoint)));
+            } else {
+                if ((MIN_SURROGATE <= c && c <= MAX_SURROGATE)
+                    && (inIx + 1 == inLimit || !isSurrogatePair(c, in.charAt(inIx + 1)))) {
+                    // We are surrogates and we're not a surrogate pair.
+                    throw new IllegalArgumentException("Unpaired surrogate at index " + inIx + " of " + inLimit);
+                }
+                // Not enough space in the output buffer.
+                throw new ArrayIndexOutOfBoundsException("Failed writing " + c + " at index " + outIx);
+            }
+        }
+
+        // All bytes have been encoded.
+        out.position((int) (outIx - address));
+    }
+
+    /**
+     * Returns the number of bytes in the UTF-8-encoded form of {@code sequence}. For a string,
+     * this method is equivalent to {@code string.getBytes(UTF_8).length}, but is more efficient in
+     * both time and space.
+     *
+     * @throws IllegalArgumentException if {@code sequence} contains ill-formed UTF-16 (unpaired
+     *                                  surrogates)
+     */
+    public static int encodedLength(CharSequence sequence) {
+        // Warning to maintainers: this implementation is highly optimized.
+        int utf16Length = sequence.length();
+        int utf8Length = utf16Length;
+        int i = 0;
+
+        // This loop optimizes for pure ASCII.
+        while (i < utf16Length && sequence.charAt(i) < 0x80) {
+            i++;
+        }
+
+        // This loop optimizes for chars less than 0x800.
+        for (; i < utf16Length; i++) {
+            char c = sequence.charAt(i);
+            if (c < 0x800) {
+                utf8Length += ((0x7f - c) >>> 31); // branch free!
+            } else {
+                utf8Length += encodedLengthGeneral(sequence, i);
+                break;
+            }
+        }
+
+        if (utf8Length < utf16Length) {
+            // Necessary and sufficient condition for overflow because of maximum 3x expansion
+            throw new IllegalArgumentException("UTF-8 length does not fit in int: " + (utf8Length + (1L << 32)));
+        }
+        return utf8Length;
+    }
+
+    private static int encodedLengthGeneral(CharSequence sequence, int start) {
+        int utf16Length = sequence.length();
+        int utf8Length = 0;
+        for (int i = start; i < utf16Length; i++) {
+            char c = sequence.charAt(i);
+            if (c < 0x800) {
+                utf8Length += (0x7f - c) >>> 31; // branch free!
+            } else {
+                utf8Length += 2;
+                // jdk7+: if (Character.isSurrogate(c)) {
+                if (Character.MIN_SURROGATE <= c && c <= Character.MAX_SURROGATE) {
+                    // Check that we have a well-formed surrogate pair.
+                    int cp = Character.codePointAt(sequence, i);
+                    if (cp < MIN_SUPPLEMENTARY_CODE_POINT) {
+                        throw new IllegalArgumentException("Unpaired surrogate at index " + i + " of " + utf16Length);
+                    }
+                    i++;
+                }
+            }
+        }
+        return utf8Length;
+    }
+
+    /**
+     * Utility methods for decoding bytes into {@link String}. Callers are responsible for extracting
+     * bytes (possibly using Unsafe methods), and checking remaining bytes. All other UTF-8 validity
+     * checks and codepoint conversion happen in this class.
+     */
+    private static class DecodeUtil {
+
+        /**
+         * Returns whether this is a single-byte codepoint (i.e., ASCII) with the form '0XXXXXXX'.
+         */
+        private static boolean isOneByte(byte b) {
+            return b >= 0;
+        }
+
+        /**
+         * Returns whether this is a two-byte codepoint with the form '10XXXXXX'.
+         */
+        private static boolean isTwoBytes(byte b) {
+            return b < (byte) 0xE0;
+        }
+
+        /**
+         * Returns whether this is a three-byte codepoint with the form '110XXXXX'.
+         */
+        private static boolean isThreeBytes(byte b) {
+            return b < (byte) 0xF0;
+        }
+
+        private static void handleOneByte(byte byte1, char[] resultArr, int resultPos) {
+            resultArr[resultPos] = (char) byte1;
+        }
+
+        private static void handleTwoBytes(byte byte1, byte byte2, char[] resultArr, int resultPos) {
+            // Simultaneously checks for illegal trailing-byte in leading position (<= '11000000') and
+            // overlong 2-byte, '11000001'.
+            if (byte1 < (byte) 0xC2 || isNotTrailingByte(byte2)) {
+                throw invalidUtf8();
+            }
+            resultArr[resultPos] = (char) (((byte1 & 0x1F) << 6) | trailingByteValue(byte2));
+        }
+
+        private static void handleThreeBytes(byte byte1, byte byte2, byte byte3, char[] resultArr, int resultPos) {
+            if (isNotTrailingByte(byte2)
+            // overlong? 5 most significant bits must not all be zero
+                || (byte1 == (byte) 0xE0 && byte2 < (byte) 0xA0)
+                // check for illegal surrogate codepoints
+                || (byte1 == (byte) 0xED && byte2 >= (byte) 0xA0) || isNotTrailingByte(byte3)) {
+                throw invalidUtf8();
+            }
+            resultArr[resultPos] = (char) (((byte1 & 0x0F) << 12) | (trailingByteValue(byte2) << 6) | trailingByteValue(byte3));
+        }
+
+        private static void handleFourBytes(byte byte1, byte byte2, byte byte3, byte byte4, char[] resultArr,
+                                            int resultPos) {
+            if (isNotTrailingByte(byte2)
+                // Check that 1 <= plane <= 16.  Tricky optimized form of:
+                //   valid 4-byte leading byte?
+                // if (byte1 > (byte) 0xF4 ||
+                //   overlong? 4 most significant bits must not all be zero
+                //     byte1 == (byte) 0xF0 && byte2 < (byte) 0x90 ||
+                //   codepoint larger than the highest code point (U+10FFFF)?
+                //     byte1 == (byte) 0xF4 && byte2 > (byte) 0x8F)
+                || (((byte1 << 28) + (byte2 - (byte) 0x90)) >> 30) != 0 || isNotTrailingByte(byte3)
+                || isNotTrailingByte(byte4)) {
+                throw invalidUtf8();
+            }
+            int codePoint = ((byte1 & 0x07) << 18) | (trailingByteValue(byte2) << 12) | (trailingByteValue(byte3) << 6)
+                            | trailingByteValue(byte4);
+            resultArr[resultPos] = DecodeUtil.highSurrogate(codePoint);
+            resultArr[resultPos + 1] = DecodeUtil.lowSurrogate(codePoint);
+        }
+
+        /**
+         * Returns whether the byte is not a valid continuation of the form '10XXXXXX'.
+         */
+        private static boolean isNotTrailingByte(byte b) {
+            return b > (byte) 0xBF;
+        }
+
+        /**
+         * Returns the actual value of the trailing byte (removes the prefix '10') for composition.
+         */
+        private static int trailingByteValue(byte b) {
+            return b & 0x3F;
+        }
+
+        private static char highSurrogate(int codePoint) {
+            return (char) ((MIN_HIGH_SURROGATE - (MIN_SUPPLEMENTARY_CODE_POINT >>> 10)) + (codePoint >>> 10));
+        }
+
+        private static char lowSurrogate(int codePoint) {
+            return (char) (MIN_LOW_SURROGATE + (codePoint & 0x3ff));
+        }
+    }
+
+    static IllegalStateException invalidUtf8() {
+        return new IllegalStateException("Message had invalid UTF-8.");
+    }
+
+    private UnsafeUtf8Util() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtil.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtil.java
new file mode 100644
index 0000000..ba0d554
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/UnsafeUtil.java
@@ -0,0 +1,629 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+import java.lang.reflect.Field;
+import java.nio.Buffer;
+import java.nio.ByteBuffer;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * For the {@link sun.misc.Unsafe} access.
+ *
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("ConstantConditions")
+public final class UnsafeUtil {
+
+    private static final Logger         LOG                       = LoggerFactory.getLogger(UnsafeUtil.class);
+
+    private static final Object         UNSAFE                    = getUnsafe0();
+
+    private static final UnsafeAccessor UNSAFE_ACCESSOR           = getUnsafeAccessor0();
+
+    private static final long           BYTE_ARRAY_BASE_OFFSET    = arrayBaseOffset(byte[].class);
+    // Micro-optimization: we can assume a scale of 1 and skip the multiply
+    // private static final long BYTE_ARRAY_INDEX_SCALE = 1;
+
+    private static final long           BOOLEAN_ARRAY_BASE_OFFSET = arrayBaseOffset(boolean[].class);
+    private static final long           BOOLEAN_ARRAY_INDEX_SCALE = arrayIndexScale(boolean[].class);
+
+    private static final long           INT_ARRAY_BASE_OFFSET     = arrayBaseOffset(int[].class);
+    private static final long           INT_ARRAY_INDEX_SCALE     = arrayIndexScale(int[].class);
+
+    private static final long           LONG_ARRAY_BASE_OFFSET    = arrayBaseOffset(long[].class);
+    private static final long           LONG_ARRAY_INDEX_SCALE    = arrayIndexScale(long[].class);
+
+    private static final long           FLOAT_ARRAY_BASE_OFFSET   = arrayBaseOffset(float[].class);
+    private static final long           FLOAT_ARRAY_INDEX_SCALE   = arrayIndexScale(float[].class);
+
+    private static final long           DOUBLE_ARRAY_BASE_OFFSET  = arrayBaseOffset(double[].class);
+    private static final long           DOUBLE_ARRAY_INDEX_SCALE  = arrayIndexScale(double[].class);
+
+    private static final long           OBJECT_ARRAY_BASE_OFFSET  = arrayBaseOffset(Object[].class);
+    private static final long           OBJECT_ARRAY_INDEX_SCALE  = arrayIndexScale(Object[].class);
+
+    private static final long           BUFFER_ADDRESS_OFFSET     = objectFieldOffset(bufferAddressField());
+
+    private static final long           STRING_VALUE_OFFSET       = objectFieldOffset(stringValueField());
+
+    /**
+     * Whether or not can use the unsafe api.
+     */
+    public static boolean hasUnsafe() {
+        return UNSAFE != null;
+    }
+
+    /**
+     * Get a {@link UnsafeAccessor} appropriate for the platform.
+     */
+    public static UnsafeAccessor getUnsafeAccessor() {
+        return UNSAFE_ACCESSOR;
+    }
+
+    public static byte getByte(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getByte(target, offset);
+    }
+
+    public static void putByte(final Object target, final long offset, final byte value) {
+        UNSAFE_ACCESSOR.putByte(target, offset, value);
+    }
+
+    public static int getInt(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getInt(target, offset);
+    }
+
+    public static void putInt(final Object target, final long offset, final int value) {
+        UNSAFE_ACCESSOR.putInt(target, offset, value);
+    }
+
+    public static long getLong(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getLong(target, offset);
+    }
+
+    public static void putLong(final Object target, final long offset, final long value) {
+        UNSAFE_ACCESSOR.putLong(target, offset, value);
+    }
+
+    public static boolean getBoolean(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getBoolean(target, offset);
+    }
+
+    public static void putBoolean(final Object target, final long offset, final boolean value) {
+        UNSAFE_ACCESSOR.putBoolean(target, offset, value);
+    }
+
+    public static float getFloat(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getFloat(target, offset);
+    }
+
+    public static void putFloat(final Object target, final long offset, final float value) {
+        UNSAFE_ACCESSOR.putFloat(target, offset, value);
+    }
+
+    public static double getDouble(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getDouble(target, offset);
+    }
+
+    public static void putDouble(final Object target, final long offset, final double value) {
+        UNSAFE_ACCESSOR.putDouble(target, offset, value);
+    }
+
+    public static Object getObject(final Object target, final long offset) {
+        return UNSAFE_ACCESSOR.getObject(target, offset);
+    }
+
+    public static void putObject(final Object target, final long offset, final Object value) {
+        UNSAFE_ACCESSOR.putObject(target, offset, value);
+    }
+
+    public static byte getByte(final byte[] target, final long index) {
+        return UNSAFE_ACCESSOR.getByte(target, BYTE_ARRAY_BASE_OFFSET + index);
+    }
+
+    public static void putByte(final byte[] target, final long index, final byte value) {
+        UNSAFE_ACCESSOR.putByte(target, BYTE_ARRAY_BASE_OFFSET + index, value);
+    }
+
+    public static int getInt(final int[] target, final long index) {
+        return UNSAFE_ACCESSOR.getInt(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putInt(final int[] target, final long index, final int value) {
+        UNSAFE_ACCESSOR.putInt(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static long getLong(final long[] target, final long index) {
+        return UNSAFE_ACCESSOR.getLong(target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putLong(final long[] target, final long index, final long value) {
+        UNSAFE_ACCESSOR.putLong(target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static boolean getBoolean(final boolean[] target, final long index) {
+        return UNSAFE_ACCESSOR.getBoolean(target, BOOLEAN_ARRAY_BASE_OFFSET + (index * BOOLEAN_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putBoolean(final boolean[] target, final long index, final boolean value) {
+        UNSAFE_ACCESSOR.putBoolean(target, BOOLEAN_ARRAY_BASE_OFFSET + (index * BOOLEAN_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static float getFloat(final float[] target, final long index) {
+        return UNSAFE_ACCESSOR.getFloat(target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putFloat(final float[] target, final long index, final float value) {
+        UNSAFE_ACCESSOR.putFloat(target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static double getDouble(final double[] target, final long index) {
+        return UNSAFE_ACCESSOR.getDouble(target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putDouble(final double[] target, final long index, final double value) {
+        UNSAFE_ACCESSOR.putDouble(target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static Object getObject(final Object[] target, final long index) {
+        return UNSAFE_ACCESSOR.getObject(target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putObject(final Object[] target, final long index, final Object value) {
+        UNSAFE_ACCESSOR.putObject(target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static byte getByte(final long address) {
+        return UNSAFE_ACCESSOR.getByte(address);
+    }
+
+    public static void putByte(final long address, final byte value) {
+        UNSAFE_ACCESSOR.putByte(address, value);
+    }
+
+    public static int getInt(final long address) {
+        return UNSAFE_ACCESSOR.getInt(address);
+    }
+
+    public static void putInt(final long address, final int value) {
+        UNSAFE_ACCESSOR.putInt(address, value);
+    }
+
+    public static long getLong(final long address) {
+        return UNSAFE_ACCESSOR.getLong(address);
+    }
+
+    public static void putLong(final long address, final long value) {
+        UNSAFE_ACCESSOR.putLong(address, value);
+    }
+
+    public static byte getByteVolatile(final byte[] target, final long index) {
+        return UNSAFE_ACCESSOR.getByteVolatile(target, BYTE_ARRAY_BASE_OFFSET + index);
+    }
+
+    public static void putByteVolatile(final byte[] target, final long index, final byte value) {
+        UNSAFE_ACCESSOR.putByteVolatile(target, BYTE_ARRAY_BASE_OFFSET + index, value);
+    }
+
+    public static int getIntVolatile(final int[] target, final long index) {
+        return UNSAFE_ACCESSOR.getIntVolatile(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putIntVolatile(final int[] target, final long index, final int value) {
+        UNSAFE_ACCESSOR.putIntVolatile(target, INT_ARRAY_BASE_OFFSET + (index * INT_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static long getLongVolatile(final long[] target, final long index) {
+        return UNSAFE_ACCESSOR.getLongVolatile(target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putLongVolatile(final long[] target, final long index, final long value) {
+        UNSAFE_ACCESSOR.putLongVolatile(target, LONG_ARRAY_BASE_OFFSET + (index * LONG_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static boolean getBooleanVolatile(final boolean[] target, final long index) {
+        return UNSAFE_ACCESSOR.getBooleanVolatile(target, BOOLEAN_ARRAY_BASE_OFFSET
+                                                          + (index * BOOLEAN_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putBooleanVolatile(final boolean[] target, final long index, final boolean value) {
+        UNSAFE_ACCESSOR.putBooleanVolatile(target, BOOLEAN_ARRAY_BASE_OFFSET + (index * BOOLEAN_ARRAY_INDEX_SCALE),
+            value);
+    }
+
+    public static float getFloatVolatile(final float[] target, final long index) {
+        return UNSAFE_ACCESSOR.getFloatVolatile(target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putFloatVolatile(final float[] target, final long index, final float value) {
+        UNSAFE_ACCESSOR.putFloatVolatile(target, FLOAT_ARRAY_BASE_OFFSET + (index * FLOAT_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static double getDoubleVolatile(final double[] target, final long index) {
+        return UNSAFE_ACCESSOR.getDoubleVolatile(target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putDoubleVolatile(final double[] target, final long index, final double value) {
+        UNSAFE_ACCESSOR.putDoubleVolatile(target, DOUBLE_ARRAY_BASE_OFFSET + (index * DOUBLE_ARRAY_INDEX_SCALE), value);
+    }
+
+    public static Object getObjectVolatile(final Object[] target, final long index) {
+        return UNSAFE_ACCESSOR.getObjectVolatile(target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE));
+    }
+
+    public static void putObjectVolatile(final Object[] target, final long index, final Object value) {
+        UNSAFE_ACCESSOR.putObjectVolatile(target, OBJECT_ARRAY_BASE_OFFSET + (index * OBJECT_ARRAY_INDEX_SCALE), value);
+    }
+
+    /**
+     * Reports the offset of the first element in the storage allocation of a
+     * given array class.
+     */
+    public static int arrayBaseOffset(final Class clazz) {
+        return hasUnsafe() ? UNSAFE_ACCESSOR.arrayBaseOffset(clazz) : -1;
+    }
+
+    /**
+     * Reports the scale factor for addressing elements in the storage
+     * allocation of a given array class.
+     */
+    public static int arrayIndexScale(final Class clazz) {
+        return hasUnsafe() ? UNSAFE_ACCESSOR.arrayIndexScale(clazz) : -1;
+    }
+
+    /**
+     * Returns the offset of the provided field, or {@code -1} if {@code sun.misc.Unsafe} is not
+     * available.
+     */
+    public static long objectFieldOffset(final Field field) {
+        return field == null || hasUnsafe() ? UNSAFE_ACCESSOR.objectFieldOffset(field) : -1;
+    }
+
+    /**
+     * Returns the offset of the provided class and fieldName, or {@code -1} if {@code sun.misc.Unsafe} is not
+     * available.
+     */
+    public static long objectFieldOffset(final Class clazz, final String fieldName) {
+        try {
+            return objectFieldOffset(clazz.getDeclaredField(fieldName));
+        } catch (final NoSuchFieldException e) {
+            UNSAFE_ACCESSOR.throwException(e);
+        }
+        return -1; // never get here
+    }
+
+    /**
+     * Gets the offset of the {@code address} field of the given
+     * direct {@link ByteBuffer}.
+     */
+    public static long addressOffset(final ByteBuffer buffer) {
+        return UNSAFE_ACCESSOR.getLong(buffer, BUFFER_ADDRESS_OFFSET);
+    }
+
+    public static void throwException(final Throwable t) {
+        UNSAFE_ACCESSOR.throwException(t);
+    }
+
+    /**
+     * Returns a new {@link String} backed by the given {@code chars}.
+     * The char array should not be mutated any more after calling
+     * this function.
+     */
+    public static String moveToString(final char[] chars) {
+        if (STRING_VALUE_OFFSET == -1) {
+            // In the off-chance that this JDK does not implement String as we'd expect, just do a copy.
+            return new String(chars);
+        }
+        final String str;
+        try {
+            str = (String) UNSAFE_ACCESSOR.allocateInstance(String.class);
+        } catch (final InstantiationException e) {
+            // This should never happen, but return a copy as a fallback just in case.
+            return new String(chars);
+        }
+        UNSAFE_ACCESSOR.putObject(str, STRING_VALUE_OFFSET, chars);
+        return str;
+    }
+
+    /**
+     * Returns the system {@link ClassLoader}.
+     */
+    public static ClassLoader getSystemClassLoader() {
+        if (System.getSecurityManager() == null) {
+            return ClassLoader.getSystemClassLoader();
+        } else {
+            return AccessController.doPrivileged((PrivilegedAction) ClassLoader::getSystemClassLoader);
+        }
+    }
+
+    /**
+     * Finds the address field within a direct {@link Buffer}.
+     */
+    private static Field bufferAddressField() {
+        return field(Buffer.class, "address", long.class);
+    }
+
+    /**
+     * Finds the value field within a {@link String}.
+     */
+    private static Field stringValueField() {
+        return field(String.class, "value", char[].class);
+    }
+
+    /**
+     * Gets the field with the given name within the class, or
+     * {@code null} if not found. If found, the field is made accessible.
+     */
+    private static Field field(final Class clazz, final String fieldName, final Class expectedType) {
+        Field field;
+        try {
+            field = clazz.getDeclaredField(fieldName);
+            field.setAccessible(true);
+            if (!field.getType().equals(expectedType)) {
+                return null;
+            }
+        } catch (final Throwable t) {
+            // Failed to access the fields.
+            field = null;
+        }
+        return field;
+    }
+
+    private static UnsafeAccessor getUnsafeAccessor0() {
+        return hasUnsafe() ? new UnsafeAccessor(UNSAFE) : null;
+    }
+
+    private static Object getUnsafe0() {
+        Object unsafe;
+        try {
+            final Class unsafeClass = Class.forName("sun.misc.Unsafe");
+            final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+            unsafeField.setAccessible(true);
+            unsafe = unsafeField.get(null);
+        } catch (final Throwable t) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("sun.misc.Unsafe.theUnsafe: unavailable.", t);
+            }
+            unsafe = null;
+        }
+        return unsafe;
+    }
+
+    public static class UnsafeAccessor {
+
+        private final sun.misc.Unsafe unsafe;
+
+        UnsafeAccessor(Object unsafe) {
+            this.unsafe = (sun.misc.Unsafe) unsafe;
+        }
+
+        /**
+         * Returns the {@link sun.misc.Unsafe}'s instance.
+         */
+        public sun.misc.Unsafe getUnsafe() {
+            return unsafe;
+        }
+
+        public byte getByte(final Object target, final long offset) {
+            return this.unsafe.getByte(target, offset);
+        }
+
+        public void putByte(final Object target, final long offset, final byte value) {
+            this.unsafe.putByte(target, offset, value);
+        }
+
+        public short getShort(final Object target, final long offset) {
+            return this.unsafe.getShort(target, offset);
+        }
+
+        public void putShort(final Object target, final long offset, final short value) {
+            this.unsafe.putShort(target, offset, value);
+        }
+
+        public int getInt(final Object target, final long offset) {
+            return this.unsafe.getInt(target, offset);
+        }
+
+        public void putInt(final Object target, final long offset, final int value) {
+            this.unsafe.putInt(target, offset, value);
+        }
+
+        public long getLong(final Object target, final long offset) {
+            return this.unsafe.getLong(target, offset);
+        }
+
+        public void putLong(final Object target, final long offset, final long value) {
+            this.unsafe.putLong(target, offset, value);
+        }
+
+        public boolean getBoolean(final Object target, final long offset) {
+            return this.unsafe.getBoolean(target, offset);
+        }
+
+        public void putBoolean(final Object target, final long offset, final boolean value) {
+            this.unsafe.putBoolean(target, offset, value);
+        }
+
+        public float getFloat(final Object target, final long offset) {
+            return this.unsafe.getFloat(target, offset);
+        }
+
+        public void putFloat(final Object target, final long offset, final float value) {
+            this.unsafe.putFloat(target, offset, value);
+        }
+
+        public double getDouble(final Object target, final long offset) {
+            return this.unsafe.getDouble(target, offset);
+        }
+
+        public void putDouble(final Object target, final long offset, final double value) {
+            this.unsafe.putDouble(target, offset, value);
+        }
+
+        public Object getObject(final Object target, final long offset) {
+            return this.unsafe.getObject(target, offset);
+        }
+
+        public void putObject(final Object target, final long offset, final Object value) {
+            this.unsafe.putObject(target, offset, value);
+        }
+
+        public byte getByte(final long address) {
+            return this.unsafe.getByte(address);
+        }
+
+        public void putByte(final long address, final byte value) {
+            this.unsafe.putByte(address, value);
+        }
+
+        public short getShort(final long address) {
+            return this.unsafe.getShort(address);
+        }
+
+        public void putShort(final long address, final short value) {
+            this.unsafe.putShort(address, value);
+        }
+
+        public int getInt(final long address) {
+            return this.unsafe.getInt(address);
+        }
+
+        public void putInt(final long address, final int value) {
+            this.unsafe.putInt(address, value);
+        }
+
+        public long getLong(final long address) {
+            return this.unsafe.getLong(address);
+        }
+
+        public void putLong(final long address, final long value) {
+            this.unsafe.putLong(address, value);
+        }
+
+        public void copyMemory(final Object srcBase, final long srcOffset, final Object dstBase, final long dstOffset,
+                               final long bytes) {
+            this.unsafe.copyMemory(srcBase, srcOffset, dstBase, dstOffset, bytes);
+        }
+
+        public void copyMemory(final long srcAddress, final long dstAddress, final long bytes) {
+            this.unsafe.copyMemory(srcAddress, dstAddress, bytes);
+        }
+
+        public byte getByteVolatile(final Object target, final long offset) {
+            return this.unsafe.getByteVolatile(target, offset);
+        }
+
+        public void putByteVolatile(final Object target, final long offset, final byte value) {
+            this.unsafe.putByteVolatile(target, offset, value);
+        }
+
+        public short getShortVolatile(final Object target, final long offset) {
+            return this.unsafe.getShortVolatile(target, offset);
+        }
+
+        public void putShortVolatile(final Object target, final long offset, final short value) {
+            this.unsafe.putShortVolatile(target, offset, value);
+        }
+
+        public int getIntVolatile(final Object target, final long offset) {
+            return this.unsafe.getIntVolatile(target, offset);
+        }
+
+        public void putIntVolatile(final Object target, final long offset, final int value) {
+            this.unsafe.putIntVolatile(target, offset, value);
+        }
+
+        public long getLongVolatile(final Object target, final long offset) {
+            return this.unsafe.getLongVolatile(target, offset);
+        }
+
+        public void putLongVolatile(final Object target, final long offset, final long value) {
+            this.unsafe.putLongVolatile(target, offset, value);
+        }
+
+        public boolean getBooleanVolatile(final Object target, final long offset) {
+            return this.unsafe.getBooleanVolatile(target, offset);
+        }
+
+        public void putBooleanVolatile(final Object target, final long offset, final boolean value) {
+            this.unsafe.putBooleanVolatile(target, offset, value);
+        }
+
+        public float getFloatVolatile(final Object target, final long offset) {
+            return this.unsafe.getFloatVolatile(target, offset);
+        }
+
+        public void putFloatVolatile(final Object target, final long offset, final float value) {
+            this.unsafe.putFloatVolatile(target, offset, value);
+        }
+
+        public double getDoubleVolatile(final Object target, final long offset) {
+            return this.unsafe.getDoubleVolatile(target, offset);
+        }
+
+        public void putDoubleVolatile(final Object target, final long offset, final double value) {
+            this.unsafe.putDoubleVolatile(target, offset, value);
+        }
+
+        public Object getObjectVolatile(final Object target, final long offset) {
+            return this.unsafe.getObjectVolatile(target, offset);
+        }
+
+        public void putObjectVolatile(final Object target, final long offset, final Object value) {
+            this.unsafe.putObjectVolatile(target, offset, value);
+        }
+
+        /**
+         * Reports the offset of the first element in the storage allocation of a
+         * given array class.
+         */
+        public int arrayBaseOffset(final Class clazz) {
+            return this.unsafe != null ? this.unsafe.arrayBaseOffset(clazz) : -1;
+        }
+
+        /**
+         * Reports the scale factor for addressing elements in the storage
+         * allocation of a given array class.
+         */
+        public int arrayIndexScale(final Class clazz) {
+            return this.unsafe != null ? this.unsafe.arrayIndexScale(clazz) : -1;
+        }
+
+        /**
+         * Returns the offset of the provided field, or {@code -1} if {@code sun.misc.Unsafe} is not
+         * available.
+         */
+        public long objectFieldOffset(final Field field) {
+            return field == null || this.unsafe == null ? -1 : this.unsafe.objectFieldOffset(field);
+        }
+
+        public Object allocateInstance(final Class clazz) throws InstantiationException {
+            return this.unsafe.allocateInstance(clazz);
+        }
+
+        public void throwException(final Throwable t) {
+            this.unsafe.throwException(t);
+        }
+    }
+
+    private UnsafeUtil() {
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/Updaters.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/Updaters.java
new file mode 100644
index 0000000..3732d41
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/internal/Updaters.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.internal;
+
+/**
+ * Sometime instead of reflection, better performance.
+ *
+ * @author jiachun.fjc
+ */
+public class Updaters {
+
+    /**
+     * Creates and returns an updater for objects with the given field.
+     *
+     * @param tClass    the class of the objects holding the field.
+     * @param fieldName the name of the field to be updated.
+     */
+    public static  IntegerFieldUpdater newIntegerFieldUpdater(final Class tClass,
+                                                                    final String fieldName) {
+        try {
+            if (UnsafeUtil.hasUnsafe()) {
+                return new UnsafeIntegerFieldUpdater<>(UnsafeUtil.getUnsafeAccessor().getUnsafe(), tClass, fieldName);
+            } else {
+                return new ReflectionIntegerFieldUpdater<>(tClass, fieldName);
+            }
+        } catch (final Throwable t) {
+            throw new RuntimeException(t);
+        }
+    }
+
+    /**
+     * Creates and returns an updater for objects with the given field.
+     *
+     * @param tClass    the class of the objects holding the field.
+     * @param fieldName the name of the field to be updated.
+     */
+    public static  LongFieldUpdater newLongFieldUpdater(final Class tClass, final String fieldName) {
+        try {
+            if (UnsafeUtil.hasUnsafe()) {
+                return new UnsafeLongFieldUpdater<>(UnsafeUtil.getUnsafeAccessor().getUnsafe(), tClass, fieldName);
+            } else {
+                return new ReflectionLongFieldUpdater<>(tClass, fieldName);
+            }
+        } catch (final Throwable t) {
+            throw new RuntimeException(t);
+        }
+    }
+
+    /**
+     * Creates and returns an updater for objects with the given field.
+     *
+     * @param tClass    the class of the objects holding the field.
+     * @param fieldName the name of the field to be updated.
+     */
+    public static  ReferenceFieldUpdater newReferenceFieldUpdater(final Class tClass,
+                                                                              final String fieldName) {
+        try {
+            if (UnsafeUtil.hasUnsafe()) {
+                return new UnsafeReferenceFieldUpdater<>(UnsafeUtil.getUnsafeAccessor().getUnsafe(), tClass, fieldName);
+            } else {
+                return new ReflectionReferenceFieldUpdater<>(tClass, fieldName);
+            }
+        } catch (final Throwable t) {
+            throw new RuntimeException(t);
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultRaftTimerFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultRaftTimerFactory.java
new file mode 100644
index 0000000..68ec4fe
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultRaftTimerFactory.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alipay.sofa.jraft.core.Scheduler;
+import com.alipay.sofa.jraft.core.TimerManager;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.SPI;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SPI
+public class DefaultRaftTimerFactory implements RaftTimerFactory {
+
+    private static final String             GLOBAL_ELECTION_TIMER_WORKERS  = "jraft.timer.global_election_timer_workers";
+    private static final String             GLOBAL_VOTE_TIMER_WORKERS      = "jraft.timer.global_vote_timer_workers";
+    private static final String             GLOBAL_STEP_DOWN_TIMER_WORKERS = "jraft.timer.global_step_down_timer_workers";
+    private static final String             GLOBAL_SNAPSHOT_TIMER_WORKERS  = "jraft.timer.global_snapshot_timer_workers";
+    private static final String             GLOBAL_SCHEDULER_WORKERS       = "jraft.timer.global_scheduler_workers";
+
+    private static final TimerSharedRef     ELECTION_TIMER_REF             = new TimerSharedRef(
+                                                                               SystemPropertyUtil.getInt(
+                                                                                   GLOBAL_ELECTION_TIMER_WORKERS,
+                                                                                   Utils.cpus()),
+                                                                               "JRaft-Global-ElectionTimer");
+    private static final TimerSharedRef     VOTE_TIMER_REF                 = new TimerSharedRef(
+                                                                               SystemPropertyUtil.getInt(
+                                                                                   GLOBAL_VOTE_TIMER_WORKERS,
+                                                                                   Utils.cpus()),
+                                                                               "JRaft-Global-VoteTimer");
+    private static final TimerSharedRef     STEP_DOWN_TIMER_REF            = new TimerSharedRef(
+                                                                               SystemPropertyUtil.getInt(
+                                                                                   GLOBAL_STEP_DOWN_TIMER_WORKERS,
+                                                                                   Utils.cpus()),
+                                                                               "JRaft-Global-StepDownTimer");
+    private static final TimerSharedRef     SNAPSHOT_TIMER_REF             = new TimerSharedRef(
+                                                                               SystemPropertyUtil.getInt(
+                                                                                   GLOBAL_SNAPSHOT_TIMER_WORKERS,
+                                                                                   Utils.cpus()),
+                                                                               "JRaft-Global-SnapshotTimer");
+    private static final SchedulerSharedRef SCHEDULER_REF                  = new SchedulerSharedRef(
+                                                                               SystemPropertyUtil.getInt(
+                                                                                   GLOBAL_SCHEDULER_WORKERS,
+                                                                                   Utils.cpus() * 3 > 20 ? 20 : Utils
+                                                                                       .cpus() * 3),
+                                                                               "JRaft-Node-ScheduleThreadPool");
+
+    @Override
+    public Timer getElectionTimer(final boolean shared, final String name) {
+        return shared ? ELECTION_TIMER_REF.getRef() : createTimer(name);
+    }
+
+    @Override
+    public Timer getVoteTimer(final boolean shared, final String name) {
+        return shared ? VOTE_TIMER_REF.getRef() : createTimer(name);
+    }
+
+    @Override
+    public Timer getStepDownTimer(final boolean shared, final String name) {
+        return shared ? STEP_DOWN_TIMER_REF.getRef() : createTimer(name);
+    }
+
+    @Override
+    public Timer getSnapshotTimer(final boolean shared, final String name) {
+        return shared ? SNAPSHOT_TIMER_REF.getRef() : createTimer(name);
+    }
+
+    @Override
+    public Scheduler getRaftScheduler(final boolean shared, final int workerNum, final String name) {
+        return shared ? SCHEDULER_REF.getRef() : createScheduler(workerNum, name);
+    }
+
+    @Override
+    public Timer createTimer(final String name) {
+        return new HashedWheelTimer(new NamedThreadFactory(name, true), 1, TimeUnit.MILLISECONDS, 2048);
+    }
+
+    @Override
+    public Scheduler createScheduler(final int workerNum, final String name) {
+        return new TimerManager(workerNum, name);
+    }
+
+    private static abstract class Shared {
+
+        private AtomicInteger refCount = new AtomicInteger(0);
+        private AtomicBoolean started  = new AtomicBoolean(true);
+        protected final T     shared;
+
+        protected Shared(T shared) {
+            this.shared = shared;
+        }
+
+        public T getRef() {
+            if (this.started.get()) {
+                this.refCount.incrementAndGet();
+                return current();
+            }
+            throw new IllegalStateException("Shared shutdown");
+        }
+
+        public boolean isShutdown() {
+            return !this.started.get();
+        }
+
+        public abstract T current();
+
+        public boolean mayShutdown() {
+            return this.refCount.decrementAndGet() <= 0 && this.started.compareAndSet(true, false);
+        }
+    }
+
+    private static abstract class SharedRef {
+
+        private final int    workerNum;
+        private final String name;
+        private Shared    shared;
+
+        public SharedRef(int workerNum, String name) {
+            this.workerNum = workerNum;
+            this.name = name;
+        }
+
+        public synchronized T getRef() {
+            if (this.shared == null || this.shared.isShutdown()) {
+                this.shared = create(this.workerNum, this.name);
+            }
+            return this.shared.getRef();
+        }
+
+        public abstract Shared create(final int workerNum, final String name);
+    }
+
+    private static class TimerSharedRef extends SharedRef {
+
+        public TimerSharedRef(int workerNum, String name) {
+            super(workerNum, name);
+        }
+
+        @Override
+        public Shared create(final int workerNum, final String name) {
+            return new SharedTimer(new DefaultTimer(workerNum, name));
+        }
+    }
+
+    private static class SharedTimer extends Shared implements Timer {
+
+        protected SharedTimer(Timer shared) {
+            super(shared);
+        }
+
+        @Override
+        public SharedTimer current() {
+            return this;
+        }
+
+        @Override
+        public Timeout newTimeout(final TimerTask task, final long delay, final TimeUnit unit) {
+            return this.shared.newTimeout(task, delay, unit);
+        }
+
+        @Override
+        public Set stop() {
+            if (mayShutdown()) {
+                return this.shared.stop();
+            }
+            return Collections.emptySet();
+        }
+    }
+
+    private static class SchedulerSharedRef extends SharedRef {
+
+        public SchedulerSharedRef(int workerNum, String name) {
+            super(workerNum, name);
+        }
+
+        @Override
+        public Shared create(final int workerNum, final String name) {
+            return new SharedScheduler(new TimerManager(workerNum, name));
+        }
+    }
+
+    private static class SharedScheduler extends Shared implements Scheduler {
+
+        protected SharedScheduler(Scheduler shared) {
+            super(shared);
+        }
+
+        @Override
+        public Scheduler current() {
+            return this;
+        }
+
+        @Override
+        public ScheduledFuture schedule(final Runnable command, final long delay, final TimeUnit unit) {
+            return this.shared.schedule(command, delay, unit);
+        }
+
+        @Override
+        public ScheduledFuture scheduleAtFixedRate(final Runnable command, final long initialDelay,
+                                                      final long period, final TimeUnit unit) {
+            return this.shared.scheduleAtFixedRate(command, initialDelay, period, unit);
+        }
+
+        @Override
+        public ScheduledFuture scheduleWithFixedDelay(final Runnable command, final long initialDelay,
+                                                         final long delay, final TimeUnit unit) {
+            return this.shared.scheduleWithFixedDelay(command, initialDelay, delay, unit);
+        }
+
+        @Override
+        public void shutdown() {
+            if (mayShutdown()) {
+                this.shared.shutdown();
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultTimer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultTimer.java
new file mode 100644
index 0000000..c91739e
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/DefaultTimer.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ * @author jiachun.fjc
+ */
+public class DefaultTimer implements Timer {
+
+    private final ScheduledExecutorService scheduledExecutorService;
+
+    public DefaultTimer(int workerNum, String name) {
+        this.scheduledExecutorService = ThreadPoolUtil.newScheduledBuilder() //
+            .coreThreads(workerNum) //
+            .poolName(name) //
+            .enableMetric(true) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .build();
+    }
+
+    @Override
+    public Timeout newTimeout(final TimerTask task, final long delay, final TimeUnit unit) {
+        Requires.requireNonNull(task, "task");
+        Requires.requireNonNull(unit, "unit");
+
+        final TimeoutTask timeoutTask = new TimeoutTask(task);
+        final ScheduledFuture future = this.scheduledExecutorService.schedule(new TimeoutTask(task), delay, unit);
+        timeoutTask.setFuture(future);
+        return timeoutTask.getTimeout();
+    }
+
+    @Override
+    public Set stop() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.scheduledExecutorService);
+        return Collections.emptySet();
+    }
+
+    private class TimeoutTask implements Runnable {
+
+        private final TimerTask             task;
+        private final Timeout               timeout;
+        private volatile ScheduledFuture future;
+
+        private TimeoutTask(TimerTask task) {
+            this.task = task;
+            this.timeout = new Timeout() {
+
+                @Override
+                public Timer timer() {
+                    return DefaultTimer.this;
+                }
+
+                @Override
+                public TimerTask task() {
+                    return task;
+                }
+
+                @Override
+                public boolean isExpired() {
+                    return false; // never use
+                }
+
+                @Override
+                public boolean isCancelled() {
+                    final ScheduledFuture f = future;
+                    return f != null && f.isCancelled();
+                }
+
+                @Override
+                public boolean cancel() {
+                    final ScheduledFuture f = future;
+                    return f != null && f.cancel(false);
+                }
+            };
+        }
+
+        public Timeout getTimeout() {
+            return timeout;
+        }
+
+        public ScheduledFuture getFuture() {
+            return future;
+        }
+
+        public void setFuture(ScheduledFuture future) {
+            this.future = future;
+        }
+
+        @Override
+        public void run() {
+            try {
+                this.task.run(this.timeout);
+            } catch (final Throwable ignored) {
+                // never get here
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/HashedWheelTimer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/HashedWheelTimer.java
new file mode 100644
index 0000000..86126c2
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/HashedWheelTimer.java
@@ -0,0 +1,758 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 
Implementation Details

+ * 

+ *
+ * Forked from Netty.
+ */
+public class HashedWheelTimer implements Timer {
+
+    private static final Logger                                      LOG                    = LoggerFactory
+                                                                                                .getLogger(HashedWheelTimer.class);
+
+    private static final int                                         INSTANCE_COUNT_LIMIT   = 256;
+    private static final AtomicInteger                               instanceCounter        = new AtomicInteger();
+    private static final AtomicBoolean                               warnedTooManyInstances = new AtomicBoolean();
+
+    private static final AtomicIntegerFieldUpdater workerStateUpdater     = AtomicIntegerFieldUpdater
+                                                                                                .newUpdater(
+                                                                                                    HashedWheelTimer.class,
+                                                                                                    "workerState");
+
+    private final Worker                                             worker                 = new Worker();
+    private final Thread                                             workerThread;
+
+    public static final int                                          WORKER_STATE_INIT      = 0;
+    public static final int                                          WORKER_STATE_STARTED   = 1;
+    public static final int                                          WORKER_STATE_SHUTDOWN  = 2;
+    @SuppressWarnings({ "unused", "FieldMayBeFinal" })
+    private volatile int                                             workerState;                                                  // 0 - init, 1 - started, 2 - shut down
+
+    private final long                                               tickDuration;
+    private final HashedWheelBucket[]                                wheel;
+    private final int                                                mask;
+    private final CountDownLatch                                     startTimeInitialized   = new CountDownLatch(1);
+    private final Queue                          timeouts               = new ConcurrentLinkedQueue<>();
+    private final Queue                          cancelledTimeouts      = new ConcurrentLinkedQueue<>();
+    private final AtomicLong                                         pendingTimeouts        = new AtomicLong(0);
+    private final long                                               maxPendingTimeouts;
+
+    private volatile long                                            startTime;
+
+    /**
+     * Creates a new timer with the default thread factory
+     * ({@link Executors#defaultThreadFactory()}), default tick duration, and
+     * default number of ticks per wheel.
+     */
+    public HashedWheelTimer() {
+        this(Executors.defaultThreadFactory());
+    }
+
+    /**
+     * Creates a new timer with the default thread factory
+     * ({@link Executors#defaultThreadFactory()}) and default number of ticks
+     * per wheel.
+     *
+     * @param tickDuration the duration between tick
+     * @param unit         the time unit of the {@code tickDuration}
+     * @throws NullPointerException     if {@code unit} is {@code null}
+     * @throws IllegalArgumentException if {@code tickDuration} is <= 0
+     */
+    public HashedWheelTimer(long tickDuration, TimeUnit unit) {
+        this(Executors.defaultThreadFactory(), tickDuration, unit);
+    }
+
+    /**
+     * Creates a new timer with the default thread factory
+     * ({@link Executors#defaultThreadFactory()}).
+     *
+     * @param tickDuration  the duration between tick
+     * @param unit          the time unit of the {@code tickDuration}
+     * @param ticksPerWheel the size of the wheel
+     * @throws NullPointerException     if {@code unit} is {@code null}
+     * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0
+     */
+    public HashedWheelTimer(long tickDuration, TimeUnit unit, int ticksPerWheel) {
+        this(Executors.defaultThreadFactory(), tickDuration, unit, ticksPerWheel);
+    }
+
+    /**
+     * Creates a new timer with the default tick duration and default number of
+     * ticks per wheel.
+     *
+     * @param threadFactory a {@link ThreadFactory} that creates a
+     *                      background {@link Thread} which is dedicated to
+     *                      {@link TimerTask} execution.
+     * @throws NullPointerException if {@code threadFactory} is {@code null}
+     */
+    public HashedWheelTimer(ThreadFactory threadFactory) {
+        this(threadFactory, 100, TimeUnit.MILLISECONDS);
+    }
+
+    /**
+     * Creates a new timer with the default number of ticks per wheel.
+     *
+     * @param threadFactory a {@link ThreadFactory} that creates a
+     *                      background {@link Thread} which is dedicated to
+     *                      {@link TimerTask} execution.
+     * @param tickDuration  the duration between tick
+     * @param unit          the time unit of the {@code tickDuration}
+     * @throws NullPointerException     if either of {@code threadFactory} and {@code unit} is {@code null}
+     * @throws IllegalArgumentException if {@code tickDuration} is <= 0
+     */
+    public HashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit) {
+        this(threadFactory, tickDuration, unit, 512);
+    }
+
+    /**
+     * Creates a new timer.
+     *
+     * @param threadFactory a {@link ThreadFactory} that creates a
+     *                      background {@link Thread} which is dedicated to
+     *                      {@link TimerTask} execution.
+     * @param tickDuration  the duration between tick
+     * @param unit          the time unit of the {@code tickDuration}
+     * @param ticksPerWheel the size of the wheel
+     * @throws NullPointerException     if either of {@code threadFactory} and {@code unit} is {@code null}
+     * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0
+     */
+    public HashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit, int ticksPerWheel) {
+        this(threadFactory, tickDuration, unit, ticksPerWheel, -1);
+    }
+
+    /**
+     * Creates a new timer.
+     *
+     * @param threadFactory      a {@link ThreadFactory} that creates a
+     *                           background {@link Thread} which is dedicated to
+     *                           {@link TimerTask} execution.
+     * @param tickDuration       the duration between tick
+     * @param unit               the time unit of the {@code tickDuration}
+     * @param ticksPerWheel      the size of the wheel
+     * @param maxPendingTimeouts The maximum number of pending timeouts after which call to
+     *                           {@code newTimeout} will result in
+     *                           {@link RejectedExecutionException}
+     *                           being thrown. No maximum pending timeouts limit is assumed if
+     *                           this value is 0 or negative.
+     * @throws NullPointerException     if either of {@code threadFactory} and {@code unit} is {@code null}
+     * @throws IllegalArgumentException if either of {@code tickDuration} and {@code ticksPerWheel} is <= 0
+     */
+    public HashedWheelTimer(ThreadFactory threadFactory, long tickDuration, TimeUnit unit, int ticksPerWheel,
+                            long maxPendingTimeouts) {
+
+        if (threadFactory == null) {
+            throw new NullPointerException("threadFactory");
+        }
+        if (unit == null) {
+            throw new NullPointerException("unit");
+        }
+        if (tickDuration <= 0) {
+            throw new IllegalArgumentException("tickDuration must be greater than 0: " + tickDuration);
+        }
+        if (ticksPerWheel <= 0) {
+            throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel);
+        }
+
+        // Normalize ticksPerWheel to power of two and initialize the wheel.
+        wheel = createWheel(ticksPerWheel);
+        mask = wheel.length - 1;
+
+        // Convert tickDuration to nanos.
+        this.tickDuration = unit.toNanos(tickDuration);
+
+        // Prevent overflow.
+        if (this.tickDuration >= Long.MAX_VALUE / wheel.length) {
+            throw new IllegalArgumentException(String.format(
+                "tickDuration: %d (expected: 0 < tickDuration in nanos < %d", tickDuration, Long.MAX_VALUE
+                                                                                            / wheel.length));
+        }
+        workerThread = threadFactory.newThread(worker);
+
+        this.maxPendingTimeouts = maxPendingTimeouts;
+
+        if (instanceCounter.incrementAndGet() > INSTANCE_COUNT_LIMIT
+            && warnedTooManyInstances.compareAndSet(false, true)) {
+            reportTooManyInstances();
+        }
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        try {
+            super.finalize();
+        } finally {
+            // This object is going to be GCed and it is assumed the ship has sailed to do a proper shutdown. If
+            // we have not yet shutdown then we want to make sure we decrement the active instance count.
+            if (workerStateUpdater.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) {
+                instanceCounter.decrementAndGet();
+            }
+        }
+    }
+
+    private static HashedWheelBucket[] createWheel(int ticksPerWheel) {
+        if (ticksPerWheel <= 0) {
+            throw new IllegalArgumentException("ticksPerWheel must be greater than 0: " + ticksPerWheel);
+        }
+        if (ticksPerWheel > 1073741824) {
+            throw new IllegalArgumentException("ticksPerWheel may not be greater than 2^30: " + ticksPerWheel);
+        }
+
+        ticksPerWheel = normalizeTicksPerWheel(ticksPerWheel);
+        HashedWheelBucket[] wheel = new HashedWheelBucket[ticksPerWheel];
+        for (int i = 0; i < wheel.length; i++) {
+            wheel[i] = new HashedWheelBucket();
+        }
+        return wheel;
+    }
+
+    private static int normalizeTicksPerWheel(int ticksPerWheel) {
+        int normalizedTicksPerWheel = 1;
+        while (normalizedTicksPerWheel < ticksPerWheel) {
+            normalizedTicksPerWheel <<= 1;
+        }
+        return normalizedTicksPerWheel;
+    }
+
+    /**
+     * Starts the background thread explicitly.  The background thread will
+     * start automatically on demand even if you did not call this method.
+     *
+     * @throws IllegalStateException if this timer has been
+     *                               {@linkplain #stop() stopped} already
+     */
+    public void start() {
+        switch (workerStateUpdater.get(this)) {
+            case WORKER_STATE_INIT:
+                if (workerStateUpdater.compareAndSet(this, WORKER_STATE_INIT, WORKER_STATE_STARTED)) {
+                    workerThread.start();
+                }
+                break;
+            case WORKER_STATE_STARTED:
+                break;
+            case WORKER_STATE_SHUTDOWN:
+                throw new IllegalStateException("cannot be started once stopped");
+            default:
+                throw new Error("Invalid WorkerState");
+        }
+
+        // Wait until the startTime is initialized by the worker.
+        while (startTime == 0) {
+            try {
+                startTimeInitialized.await();
+            } catch (InterruptedException ignore) {
+                // Ignore - it will be ready very soon.
+            }
+        }
+    }
+
+    @Override
+    public Set stop() {
+        if (Thread.currentThread() == workerThread) {
+            throw new IllegalStateException(HashedWheelTimer.class.getSimpleName() + ".stop() cannot be called from "
+                                            + TimerTask.class.getSimpleName());
+        }
+
+        if (!workerStateUpdater.compareAndSet(this, WORKER_STATE_STARTED, WORKER_STATE_SHUTDOWN)) {
+            // workerState can be 0 or 2 at this moment - let it always be 2.
+            if (workerStateUpdater.getAndSet(this, WORKER_STATE_SHUTDOWN) != WORKER_STATE_SHUTDOWN) {
+                instanceCounter.decrementAndGet();
+            }
+
+            return Collections.emptySet();
+        }
+
+        try {
+            boolean interrupted = false;
+            while (workerThread.isAlive()) {
+                workerThread.interrupt();
+                try {
+                    workerThread.join(100);
+                } catch (InterruptedException ignored) {
+                    interrupted = true;
+                }
+            }
+
+            if (interrupted) {
+                Thread.currentThread().interrupt();
+            }
+        } finally {
+            instanceCounter.decrementAndGet();
+        }
+        return worker.unprocessedTimeouts();
+    }
+
+    @Override
+    public Timeout newTimeout(TimerTask task, long delay, TimeUnit unit) {
+        if (task == null) {
+            throw new NullPointerException("task");
+        }
+        if (unit == null) {
+            throw new NullPointerException("unit");
+        }
+
+        long pendingTimeoutsCount = pendingTimeouts.incrementAndGet();
+
+        if (maxPendingTimeouts > 0 && pendingTimeoutsCount > maxPendingTimeouts) {
+            pendingTimeouts.decrementAndGet();
+            throw new RejectedExecutionException("Number of pending timeouts (" + pendingTimeoutsCount
+                                                 + ") is greater than or equal to maximum allowed pending "
+                                                 + "timeouts (" + maxPendingTimeouts + ")");
+        }
+
+        start();
+
+        // Add the timeout to the timeout queue which will be processed on the next tick.
+        // During processing all the queued HashedWheelTimeouts will be added to the correct HashedWheelBucket.
+        long deadline = System.nanoTime() + unit.toNanos(delay) - startTime;
+
+        // Guard against overflow.
+        if (delay > 0 && deadline < 0) {
+            deadline = Long.MAX_VALUE;
+        }
+        HashedWheelTimeout timeout = new HashedWheelTimeout(this, task, deadline);
+        timeouts.add(timeout);
+        return timeout;
+    }
+
+    /**
+     * Returns the number of pending timeouts of this {@link Timer}.
+     */
+    public long pendingTimeouts() {
+        return pendingTimeouts.get();
+    }
+
+    private static void reportTooManyInstances() {
+        String resourceType = HashedWheelTimer.class.getSimpleName();
+        LOG.error("You are creating too many {} instances.  {} is a shared resource that must be "
+                  + "reused across the JVM, so that only a few instances are created.", resourceType, resourceType);
+    }
+
+    private final class Worker implements Runnable {
+        private final Set unprocessedTimeouts = new HashSet<>();
+
+        private long               tick;
+
+        @Override
+        public void run() {
+            // Initialize the startTime.
+            startTime = System.nanoTime();
+            if (startTime == 0) {
+                // We use 0 as an indicator for the uninitialized value here, so make sure it's not 0 when initialized.
+                startTime = 1;
+            }
+
+            // Notify the other threads waiting for the initialization at start().
+            startTimeInitialized.countDown();
+
+            do {
+                final long deadline = waitForNextTick();
+                if (deadline > 0) {
+                    int idx = (int) (tick & mask);
+                    processCancelledTasks();
+                    HashedWheelBucket bucket = wheel[idx];
+                    transferTimeoutsToBuckets();
+                    bucket.expireTimeouts(deadline);
+                    tick++;
+                }
+            } while (workerStateUpdater.get(HashedWheelTimer.this) == WORKER_STATE_STARTED);
+
+            // Fill the unprocessedTimeouts so we can return them from stop() method.
+            for (HashedWheelBucket bucket : wheel) {
+                bucket.clearTimeouts(unprocessedTimeouts);
+            }
+            for (;;) {
+                HashedWheelTimeout timeout = timeouts.poll();
+                if (timeout == null) {
+                    break;
+                }
+                if (!timeout.isCancelled()) {
+                    unprocessedTimeouts.add(timeout);
+                }
+            }
+            processCancelledTasks();
+        }
+
+        private void transferTimeoutsToBuckets() {
+            // transfer only max. 100000 timeouts per tick to prevent a thread to stale the workerThread when it just
+            // adds new timeouts in a loop.
+            for (int i = 0; i < 100000; i++) {
+                HashedWheelTimeout timeout = timeouts.poll();
+                if (timeout == null) {
+                    // all processed
+                    break;
+                }
+                if (timeout.state() == HashedWheelTimeout.ST_CANCELLED) {
+                    // Was cancelled in the meantime.
+                    continue;
+                }
+
+                long calculated = timeout.deadline / tickDuration;
+                timeout.remainingRounds = (calculated - tick) / wheel.length;
+
+                final long ticks = Math.max(calculated, tick); // Ensure we don't schedule for past.
+                int stopIndex = (int) (ticks & mask);
+
+                HashedWheelBucket bucket = wheel[stopIndex];
+                bucket.addTimeout(timeout);
+            }
+        }
+
+        private void processCancelledTasks() {
+            for (;;) {
+                HashedWheelTimeout timeout = cancelledTimeouts.poll();
+                if (timeout == null) {
+                    // all processed
+                    break;
+                }
+                try {
+                    timeout.remove();
+                } catch (Throwable t) {
+                    if (LOG.isWarnEnabled()) {
+                        LOG.warn("An exception was thrown while process a cancellation task", t);
+                    }
+                }
+            }
+        }
+
+        /**
+         * Calculate goal nanoTime from startTime and current tick number,
+         * then wait until that goal has been reached.
+         *
+         * @return Long.MIN_VALUE if received a shutdown request,
+         * current time otherwise (with Long.MIN_VALUE changed by +1)
+         */
+        private long waitForNextTick() {
+            long deadline = tickDuration * (tick + 1);
+
+            for (;;) {
+                final long currentTime = System.nanoTime() - startTime;
+                long sleepTimeMs = (deadline - currentTime + 999999) / 1000000;
+
+                if (sleepTimeMs <= 0) {
+                    if (currentTime == Long.MIN_VALUE) {
+                        return -Long.MAX_VALUE;
+                    } else {
+                        return currentTime;
+                    }
+                }
+
+                // We decide to remove the original approach (as below) which used in netty for
+                // windows platform.
+                // See https://github.com/netty/netty/issues/356
+                //
+                // if (Platform.isWindows()) {
+                //     sleepTimeMs = sleepTimeMs / 10 * 10;
+                // }
+                //
+                // The above approach that make sleepTimes to be a multiple of 10ms will
+                // lead to severe spin in this loop for several milliseconds, which
+                // causes the high CPU usage.
+                // See https://github.com/sofastack/sofa-jraft/issues/311
+                //
+                // According to the regression testing on windows, we haven't reproduced the
+                // Thread.sleep() bug referenced in https://www.javamex.com/tutorials/threads/sleep_issues.shtml
+                // yet.
+                //
+                // The regression testing environment:
+                // - SOFAJRaft version: 1.2.6
+                // - JVM version (e.g. java -version): JDK 1.8.0_191
+                // - OS version: Windows 7 ultimate 64 bit
+                // - CPU: Intel(R) Core(TM) i7-2670QM CPU @ 2.20GHz (4 cores)
+
+                try {
+                    Thread.sleep(sleepTimeMs);
+                } catch (InterruptedException ignored) {
+                    if (workerStateUpdater.get(HashedWheelTimer.this) == WORKER_STATE_SHUTDOWN) {
+                        return Long.MIN_VALUE;
+                    }
+                }
+            }
+        }
+
+        public Set unprocessedTimeouts() {
+            return Collections.unmodifiableSet(unprocessedTimeouts);
+        }
+    }
+
+    private static final class HashedWheelTimeout implements Timeout {
+
+        private static final int                                           ST_INIT       = 0;
+        private static final int                                           ST_CANCELLED  = 1;
+        private static final int                                           ST_EXPIRED    = 2;
+        private static final AtomicIntegerFieldUpdater STATE_UPDATER = AtomicIntegerFieldUpdater
+                                                                                             .newUpdater(
+                                                                                                 HashedWheelTimeout.class,
+                                                                                                 "state");
+
+        private final HashedWheelTimer                                     timer;
+        private final TimerTask                                            task;
+        private final long                                                 deadline;
+
+        @SuppressWarnings({ "unused", "FieldMayBeFinal", "RedundantFieldInitialization" })
+        private volatile int                                               state         = ST_INIT;
+
+        // remainingRounds will be calculated and set by Worker.transferTimeoutsToBuckets() before the
+        // HashedWheelTimeout will be added to the correct HashedWheelBucket.
+        long                                                               remainingRounds;
+
+        // This will be used to chain timeouts in HashedWheelTimerBucket via a double-linked-list.
+        // As only the workerThread will act on it there is no need for synchronization / volatile.
+        HashedWheelTimeout                                                 next;
+        HashedWheelTimeout                                                 prev;
+
+        // The bucket to which the timeout was added
+        HashedWheelBucket                                                  bucket;
+
+        HashedWheelTimeout(HashedWheelTimer timer, TimerTask task, long deadline) {
+            this.timer = timer;
+            this.task = task;
+            this.deadline = deadline;
+        }
+
+        @Override
+        public Timer timer() {
+            return timer;
+        }
+
+        @Override
+        public TimerTask task() {
+            return task;
+        }
+
+        @Override
+        public boolean cancel() {
+            // only update the state it will be removed from HashedWheelBucket on next tick.
+            if (!compareAndSetState(ST_INIT, ST_CANCELLED)) {
+                return false;
+            }
+            // If a task should be canceled we put this to another queue which will be processed on each tick.
+            // So this means that we will have a GC latency of max. 1 tick duration which is good enough. This way
+            // we can make again use of our MpscLinkedQueue and so minimize the locking / overhead as much as possible.
+            timer.cancelledTimeouts.add(this);
+            return true;
+        }
+
+        void remove() {
+            HashedWheelBucket bucket = this.bucket;
+            if (bucket != null) {
+                bucket.remove(this);
+            } else {
+                timer.pendingTimeouts.decrementAndGet();
+            }
+        }
+
+        public boolean compareAndSetState(int expected, int state) {
+            return STATE_UPDATER.compareAndSet(this, expected, state);
+        }
+
+        public int state() {
+            return state;
+        }
+
+        @Override
+        public boolean isCancelled() {
+            return state() == ST_CANCELLED;
+        }
+
+        @Override
+        public boolean isExpired() {
+            return state() == ST_EXPIRED;
+        }
+
+        public void expire() {
+            if (!compareAndSetState(ST_INIT, ST_EXPIRED)) {
+                return;
+            }
+
+            try {
+                task.run(this);
+            } catch (Throwable t) {
+                if (LOG.isWarnEnabled()) {
+                    LOG.warn("An exception was thrown by " + TimerTask.class.getSimpleName() + '.', t);
+                }
+            }
+        }
+
+        @Override
+        public String toString() {
+            final long currentTime = System.nanoTime();
+            long remaining = deadline - currentTime + timer.startTime;
+
+            StringBuilder buf = new StringBuilder(192).append(getClass().getSimpleName()).append('(')
+                .append("deadline: ");
+            if (remaining > 0) {
+                buf.append(remaining).append(" ns later");
+            } else if (remaining < 0) {
+                buf.append(-remaining).append(" ns ago");
+            } else {
+                buf.append("now");
+            }
+
+            if (isCancelled()) {
+                buf.append(", cancelled");
+            }
+
+            return buf.append(", task: ").append(task()).append(')').toString();
+        }
+    }
+
+    /**
+     * Bucket that stores HashedWheelTimeouts. These are stored in a linked-list like datastructure to allow easy
+     * removal of HashedWheelTimeouts in the middle. Also the HashedWheelTimeout act as nodes themself and so no
+     * extra object creation is needed.
+     */
+    private static final class HashedWheelBucket {
+        // Used for the linked-list datastructure
+        private HashedWheelTimeout head;
+        private HashedWheelTimeout tail;
+
+        /**
+         * Add {@link HashedWheelTimeout} to this bucket.
+         */
+        public void addTimeout(HashedWheelTimeout timeout) {
+            assert timeout.bucket == null;
+            timeout.bucket = this;
+            if (head == null) {
+                head = tail = timeout;
+            } else {
+                tail.next = timeout;
+                timeout.prev = tail;
+                tail = timeout;
+            }
+        }
+
+        /**
+         * Expire all {@link HashedWheelTimeout}s for the given {@code deadline}.
+         */
+        public void expireTimeouts(long deadline) {
+            HashedWheelTimeout timeout = head;
+
+            // process all timeouts
+            while (timeout != null) {
+                HashedWheelTimeout next = timeout.next;
+                if (timeout.remainingRounds <= 0) {
+                    next = remove(timeout);
+                    if (timeout.deadline <= deadline) {
+                        timeout.expire();
+                    } else {
+                        // The timeout was placed into a wrong slot. This should never happen.
+                        throw new IllegalStateException(String.format("timeout.deadline (%d) > deadline (%d)",
+                            timeout.deadline, deadline));
+                    }
+                } else if (timeout.isCancelled()) {
+                    next = remove(timeout);
+                } else {
+                    timeout.remainingRounds--;
+                }
+                timeout = next;
+            }
+        }
+
+        public HashedWheelTimeout remove(HashedWheelTimeout timeout) {
+            HashedWheelTimeout next = timeout.next;
+            // remove timeout that was either processed or cancelled by updating the linked-list
+            if (timeout.prev != null) {
+                timeout.prev.next = next;
+            }
+            if (timeout.next != null) {
+                timeout.next.prev = timeout.prev;
+            }
+
+            if (timeout == head) {
+                // if timeout is also the tail we need to adjust the entry too
+                if (timeout == tail) {
+                    tail = null;
+                    head = null;
+                } else {
+                    head = next;
+                }
+            } else if (timeout == tail) {
+                // if the timeout is the tail modify the tail to be the prev node.
+                tail = timeout.prev;
+            }
+            // null out prev, next and bucket to allow for GC.
+            timeout.prev = null;
+            timeout.next = null;
+            timeout.bucket = null;
+            timeout.timer.pendingTimeouts.decrementAndGet();
+            return next;
+        }
+
+        /**
+         * Clear this bucket and return all not expired / cancelled {@link Timeout}s.
+         */
+        public void clearTimeouts(Set set) {
+            for (;;) {
+                HashedWheelTimeout timeout = pollTimeout();
+                if (timeout == null) {
+                    return;
+                }
+                if (timeout.isExpired() || timeout.isCancelled()) {
+                    continue;
+                }
+                set.add(timeout);
+            }
+        }
+
+        private HashedWheelTimeout pollTimeout() {
+            HashedWheelTimeout head = this.head;
+            if (head == null) {
+                return null;
+            }
+            HashedWheelTimeout next = head.next;
+            if (next == null) {
+                tail = this.head = null;
+            } else {
+                this.head = next;
+                next.prev = null;
+            }
+
+            // null out prev and next to allow for GC.
+            head.next = null;
+            head.prev = null;
+            head.bucket = null;
+            return head;
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/RaftTimerFactory.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/RaftTimerFactory.java
new file mode 100644
index 0000000..fba500d
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/RaftTimerFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+import com.alipay.sofa.jraft.core.Scheduler;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface RaftTimerFactory {
+
+    Timer getElectionTimer(final boolean shared, final String name);
+
+    Timer getVoteTimer(final boolean shared, final String name);
+
+    Timer getStepDownTimer(final boolean shared, final String name);
+
+    Timer getSnapshotTimer(final boolean shared, final String name);
+
+    Scheduler getRaftScheduler(final boolean shared, final int workerNum, final String name);
+
+    Timer createTimer(final String name);
+
+    Scheduler createScheduler(final int workerNum, final String name);
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timeout.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timeout.java
new file mode 100644
index 0000000..bf95f6a
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timeout.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+/**
+ * A handle associated with a {@link TimerTask} that is returned by a
+ * {@link Timer}.
+ *
+ * Forked from Netty.
+ */
+public interface Timeout {
+
+    /**
+     * Returns the {@link Timer} that created this handle.
+     */
+    Timer timer();
+
+    /**
+     * Returns the {@link TimerTask} which is associated with this handle.
+     */
+    TimerTask task();
+
+    /**
+     * Returns {@code true} if and only if the {@link TimerTask} associated
+     * with this handle has been expired.
+     */
+    boolean isExpired();
+
+    /**
+     * Returns {@code true} if and only if the {@link TimerTask} associated
+     * with this handle has been cancelled.
+     */
+    boolean isCancelled();
+
+    /**
+     * Attempts to cancel the {@link TimerTask} associated with this handle.
+     * If the task has been executed or cancelled already, it will return with
+     * no side effect.
+     *
+     * @return True if the cancellation completed successfully, otherwise false
+     */
+    boolean cancel();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timer.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timer.java
new file mode 100644
index 0000000..121f0cd
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/Timer.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+import java.util.Set;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Schedules {@link TimerTask}s for one-time future execution in a background
+ * thread.
+ *
+ * Forked from Netty.
+ */
+public interface Timer {
+
+    /**
+     * Schedules the specified {@link TimerTask} for one-time execution after
+     * the specified delay.
+     *
+     * @return a handle which is associated with the specified task
+     *
+     * @throws IllegalStateException       if this timer has been {@linkplain #stop() stopped} already
+     * @throws RejectedExecutionException if the pending timeouts are too many and creating new timeout
+     *                                    can cause instability in the system.
+     */
+    Timeout newTimeout(final TimerTask task, final long delay, final TimeUnit unit);
+
+    /**
+     * Releases all resources acquired by this {@link Timer} and cancels all
+     * tasks which were scheduled but not executed yet.
+     *
+     * @return the handles associated with the tasks which were canceled by
+     *         this method
+     */
+    Set stop();
+}
diff --git a/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/TimerTask.java b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/TimerTask.java
new file mode 100644
index 0000000..d1ad18f
--- /dev/null
+++ b/jraft-core/src/main/java/com/alipay/sofa/jraft/util/timer/TimerTask.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.util.timer;
+
+/**
+ * A task which is executed after the delay specified with
+ * Timer#newTimeout(TimerTask, long, TimeUnit).
+ *
+ * Forked from Netty.
+ */
+public interface TimerTask {
+
+    /**
+     * Executed after the delay specified with
+     * Timer#newTimeout(TimerTask, long, TimeUnit).
+     *
+     * @param timeout a handle which is associated with this task
+     */
+    void run(final Timeout timeout) throws Exception;
+}
diff --git a/jraft-core/src/main/java/com/google/protobuf/BytesCarrier.java b/jraft-core/src/main/java/com/google/protobuf/BytesCarrier.java
new file mode 100644
index 0000000..81af439
--- /dev/null
+++ b/jraft-core/src/main/java/com/google/protobuf/BytesCarrier.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.protobuf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * 
+ * @author jiachun.fjc
+ */
+public class BytesCarrier extends ByteOutput {
+
+    private byte[]  value;
+    private boolean valid;
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public boolean isValid() {
+        return valid;
+    }
+
+    @Override
+    public void write(final byte value) {
+        this.valid = false;
+    }
+
+    @Override
+    public void write(final byte[] value, final int offset, final int length) {
+        doWrite(value, offset, length);
+    }
+
+    @Override
+    public void writeLazy(final byte[] value, final int offset, final int length) {
+        doWrite(value, offset, length);
+    }
+
+    @Override
+    public void write(final ByteBuffer value) throws IOException {
+        this.valid = false;
+    }
+
+    @Override
+    public void writeLazy(final ByteBuffer value) throws IOException {
+        this.valid = false;
+    }
+
+    private void doWrite(final byte[] value, final int offset, final int length) {
+        if (this.value != null) {
+            this.valid = false;
+            return;
+        }
+        if (offset == 0 && length == value.length) {
+            this.value = value;
+            this.valid = true;
+        }
+    }
+}
diff --git a/jraft-core/src/main/java/com/google/protobuf/ZeroByteStringHelper.java b/jraft-core/src/main/java/com/google/protobuf/ZeroByteStringHelper.java
new file mode 100644
index 0000000..a89ade3
--- /dev/null
+++ b/jraft-core/src/main/java/com/google/protobuf/ZeroByteStringHelper.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.protobuf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+/**
+ * Byte string and byte buffer converter
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-May-08 2:38:42 PM
+ */
+public class ZeroByteStringHelper {
+
+    /**
+     * Wrap a byte array into a ByteString.
+     */
+    public static ByteString wrap(final byte[] bs) {
+        return ByteString.wrap(bs);
+    }
+
+    /**
+     * Wrap a byte array into a ByteString.
+     * @param bs     the byte array
+     * @param offset read start offset in array
+     * @param len    read data length
+     *@return the    result byte string.
+     */
+    public static ByteString wrap(final byte[] bs, final int offset, final int len) {
+        return ByteString.wrap(bs, offset, len);
+    }
+
+    /**
+     * Wrap a byte buffer into a ByteString.
+     */
+    public static ByteString wrap(final ByteBuffer buf) {
+        return ByteString.wrap(buf);
+    }
+
+    /**
+     * Carry the byte[] from {@link ByteString}, if failed,
+     * then call {@link ByteString#toByteArray()}.
+     *
+     * @param byteString the byteString source data
+     * @return carried bytes
+     */
+    public static byte[] getByteArray(final ByteString byteString) {
+        final BytesCarrier carrier = new BytesCarrier();
+        try {
+            byteString.writeTo(carrier);
+            if (carrier.isValid()) {
+                return carrier.getValue();
+            }
+        } catch (final IOException ignored) {
+            // ignored
+        }
+        return byteString.toByteArray();
+    }
+
+    /**
+     * Concatenate the given strings while performing various optimizations to
+     * slow the growth rate of tree depth and tree node count. The result is
+     * either a {@link com.google.protobuf.ByteString.LeafByteString} or a
+     * {@link RopeByteString} depending on which optimizations, if any, were
+     * applied.
+     *
+     * 
Small pieces of length less than {@link
+     * ByteString#CONCATENATE_BY_COPY_SIZE} may be copied by value here, as in
+     * BAP95.  Large pieces are referenced without copy.
+     *
+     * 
Most of the operation here is inspired by the now-famous paper 
+     *
+     * @param left  string on the left
+     * @param right string on the right
+     * @return concatenation representing the same sequence as the given strings
+     */
+    public static ByteString concatenate(final ByteString left, final ByteString right) {
+        return RopeByteString.concatenate(left, right);
+    }
+
+    /**
+     * @see #concatenate(ByteString, ByteString)
+     */
+    public static ByteString concatenate(final List byteBuffers) {
+        final int size = byteBuffers.size();
+        if (size == 0) {
+            return null;
+        }
+        if (size == 1) {
+            return wrap(byteBuffers.get(0));
+        }
+
+        ByteString left = null;
+        for (final ByteBuffer buf : byteBuffers) {
+            if (buf.remaining() > 0) {
+                if (left == null) {
+                    left = wrap(buf);
+                } else {
+                    left = concatenate(left, wrap(buf));
+                }
+            }
+        }
+        return left;
+    }
+}
diff --git a/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory
new file mode 100644
index 0000000..92d52a0
--- /dev/null
+++ b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.core.DefaultJRaftServiceFactory
\ No newline at end of file
diff --git a/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
new file mode 100644
index 0000000..8416bc1
--- /dev/null
+++ b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory
\ No newline at end of file
diff --git a/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
new file mode 100644
index 0000000..c41f95b
--- /dev/null
+++ b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
@@ -0,0 +1,3 @@
+com.alipay.sofa.jraft.NodeDescribeSignalHandler
+com.alipay.sofa.jraft.NodeMetricsSignalHandler
+com.alipay.sofa.jraft.ThreadPoolMetricsSignalHandler
\ No newline at end of file
diff --git a/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory
new file mode 100644
index 0000000..943fda2
--- /dev/null
+++ b/jraft-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.util.timer.DefaultRaftTimerFactory
\ No newline at end of file
diff --git a/jraft-core/src/main/resources/cli.proto b/jraft-core/src/main/resources/cli.proto
new file mode 100644
index 0000000..93a464e
--- /dev/null
+++ b/jraft-core/src/main/resources/cli.proto
@@ -0,0 +1,107 @@
+syntax="proto2";
+
+package jraft;
+import "rpc.proto";
+
+option java_package="com.alipay.sofa.jraft.rpc";
+option java_outer_classname = "CliRequests";
+
+message AddPeerRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    required string peer_id = 3;
+}
+
+message AddPeerResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message RemovePeerRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    required string peer_id = 3;
+}
+
+message RemovePeerResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message ChangePeersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string new_peers = 3;
+}
+
+message ChangePeersResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message SnapshotRequest {
+    required string group_id = 1;
+    optional string peer_id = 2;
+};
+
+message ResetPeerRequest {
+    required string group_id = 1;
+    required string peer_id = 2;
+    repeated string old_peers = 3;
+    repeated string new_peers = 4;
+}
+
+message TransferLeaderRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    optional string peer_id = 3;
+}
+
+message GetLeaderRequest {
+    required string group_id = 1;
+    optional string peer_id = 2;
+}
+
+message GetLeaderResponse {
+    required string leader_id = 1;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message GetPeersRequest {
+    required string group_id = 1;
+    optional string leader_id = 2;
+    optional bool only_alive = 3 [default = false];
+}
+
+message GetPeersResponse {
+    repeated string peers = 1;
+    repeated string learners = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message AddLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message RemoveLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message ResetLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message LearnersOpResponse {
+    repeated string old_learners = 1;
+    repeated string new_learners = 2;
+    optional ErrorResponse errorResponse = 99;
+}
diff --git a/jraft-core/src/main/resources/enum.proto b/jraft-core/src/main/resources/enum.proto
new file mode 100644
index 0000000..d8ab57b
--- /dev/null
+++ b/jraft-core/src/main/resources/enum.proto
@@ -0,0 +1,21 @@
+syntax="proto2";
+package jraft;
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "EnumOutter";
+
+enum EntryType {
+    ENTRY_TYPE_UNKNOWN = 0;
+    ENTRY_TYPE_NO_OP = 1;
+    ENTRY_TYPE_DATA = 2;
+    ENTRY_TYPE_CONFIGURATION= 3;
+};
+
+enum ErrorType {
+    ERROR_TYPE_NONE = 0;
+    ERROR_TYPE_LOG = 1;
+    ERROR_TYPE_STABLE = 2;
+    ERROR_TYPE_SNAPSHOT = 3;
+    ERROR_TYPE_STATE_MACHINE = 4;
+    ERROR_TYPE_META = 5;
+};
diff --git a/jraft-core/src/main/resources/gen.sh b/jraft-core/src/main/resources/gen.sh
new file mode 100644
index 0000000..d030056
--- /dev/null
+++ b/jraft-core/src/main/resources/gen.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+protoc -I=./  --descriptor_set_out=raft.desc --java_out=../java/ enum.proto local_file_meta.proto raft.proto local_storage.proto rpc.proto cli.proto log.proto
diff --git a/jraft-core/src/main/resources/local_file_meta.proto b/jraft-core/src/main/resources/local_file_meta.proto
new file mode 100644
index 0000000..c90a245
--- /dev/null
+++ b/jraft-core/src/main/resources/local_file_meta.proto
@@ -0,0 +1,18 @@
+syntax="proto2";
+
+package jraft;
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "LocalFileMetaOutter";
+
+
+enum FileSource {
+    FILE_SOURCE_LOCAL = 0;
+    FILE_SOURCE_REFERENCE = 1;
+}
+
+message LocalFileMeta {
+    optional bytes user_meta   = 1;
+    optional FileSource source = 2;
+    optional string checksum   = 3;
+}
diff --git a/jraft-core/src/main/resources/local_storage.proto b/jraft-core/src/main/resources/local_storage.proto
new file mode 100644
index 0000000..20ba7ff
--- /dev/null
+++ b/jraft-core/src/main/resources/local_storage.proto
@@ -0,0 +1,34 @@
+syntax="proto2";
+
+
+package jraft;
+
+import "raft.proto";
+import "local_file_meta.proto";
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "LocalStorageOutter";
+
+
+message ConfigurationPBMeta {
+    repeated string peers = 1;
+    repeated string old_peers = 2;
+};
+
+message LogPBMeta {
+    required int64 first_log_index = 1;
+};
+
+message StablePBMeta {
+    required int64 term = 1;
+    required string votedfor = 2;
+};
+
+message LocalSnapshotPbMeta {
+    message File {
+        required string name = 1;
+        optional LocalFileMeta meta = 2;
+    };
+    optional SnapshotMeta meta = 1;
+    repeated File files = 2;
+}
diff --git a/jraft-core/src/main/resources/log.proto b/jraft-core/src/main/resources/log.proto
new file mode 100644
index 0000000..b410fca
--- /dev/null
+++ b/jraft-core/src/main/resources/log.proto
@@ -0,0 +1,20 @@
+syntax="proto2";
+
+package jraft;
+
+import "enum.proto";
+
+option java_package="com.alipay.sofa.jraft.entity.codec.v2";
+option java_outer_classname = "LogOutter";
+
+message PBLogEntry {
+    required EntryType type = 1;
+    required int64 term = 2;
+    required int64 index = 3;
+    repeated bytes peers = 4;
+    repeated bytes old_peers = 5;
+    required bytes data = 6;
+    optional int64 checksum = 7;
+    repeated bytes learners = 8;
+    repeated bytes old_learners = 9;
+};
diff --git a/jraft-core/src/main/resources/raft.desc b/jraft-core/src/main/resources/raft.desc
new file mode 100644
index 0000000..6b0a544
Binary files /dev/null and b/jraft-core/src/main/resources/raft.desc differ
diff --git a/jraft-core/src/main/resources/raft.proto b/jraft-core/src/main/resources/raft.proto
new file mode 100644
index 0000000..040c2de
--- /dev/null
+++ b/jraft-core/src/main/resources/raft.proto
@@ -0,0 +1,32 @@
+syntax="proto2";
+
+package jraft;
+
+import "enum.proto";
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "RaftOutter";
+
+
+message EntryMeta {
+    required int64 term = 1;
+    required EntryType type = 2;
+    repeated string peers = 3;
+    optional int64 data_len = 4;
+    // Don't change field id of `old_peers' in the consideration of backward
+    // compatibility
+    repeated string old_peers = 5;
+    // Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+    optional int64 checksum = 6;
+    repeated string learners = 7;
+    repeated string old_learners = 8;
+};
+
+message SnapshotMeta {
+    required int64 last_included_index = 1;
+    required int64 last_included_term = 2;
+    repeated string peers = 3;
+    repeated string old_peers = 4;
+    repeated string learners = 5;
+    repeated string old_learners = 6;
+}
diff --git a/jraft-core/src/main/resources/rpc.proto b/jraft-core/src/main/resources/rpc.proto
new file mode 100644
index 0000000..87544ac
--- /dev/null
+++ b/jraft-core/src/main/resources/rpc.proto
@@ -0,0 +1,114 @@
+syntax="proto2";
+
+package jraft;
+import "raft.proto";
+
+option java_package="com.alipay.sofa.jraft.rpc";
+option java_outer_classname = "RpcRequests";
+
+message PingRequest {
+  required int64 send_timestamp = 1;
+}
+
+message ErrorResponse {
+  required int32 errorCode = 1;
+  optional string errorMsg = 2;
+}
+
+message InstallSnapshotRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required SnapshotMeta meta = 5;
+  required string uri = 6;
+};
+
+message InstallSnapshotResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message TimeoutNowRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+}
+
+message TimeoutNowResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+}
+
+message RequestVoteRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required int64 last_log_term = 5;
+  required int64 last_log_index = 6;
+  required bool pre_vote = 7;
+};
+
+message RequestVoteResponse {
+  required int64 term = 1;
+  required bool granted = 2;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message AppendEntriesRequestHeader {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+};
+
+message AppendEntriesRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required int64 prev_log_term = 5;
+  required int64 prev_log_index = 6;
+  repeated EntryMeta entries = 7;
+  required int64 committed_index = 8;
+  optional bytes data = 9;
+};
+
+message AppendEntriesResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional int64 last_log_index = 3;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message GetFileRequest {
+  required int64 reader_id = 1;
+  required string filename = 2;
+  required int64 count = 3;
+  required int64 offset = 4;
+  optional bool read_partly = 5;
+}
+
+message GetFileResponse {
+  // Data is in attachment
+  required bool eof = 1;
+  required bytes data = 2;
+  optional int64 read_size = 3;
+  optional ErrorResponse errorResponse = 99;
+}
+
+message ReadIndexRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  repeated bytes entries = 3;
+  optional string peer_id = 4;
+}
+
+message ReadIndexResponse {
+  required int64 index = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
new file mode 100644
index 0000000..1995d3b
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.core.TestCluster;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class RouteTableTest {
+
+    static final Logger  LOG     = LoggerFactory.getLogger(RouteTableTest.class);
+
+    private String       dataPath;
+
+    private TestCluster  cluster;
+    private final String groupId = "RouteTableTest";
+
+    CliClientServiceImpl cliClientService;
+
+    @Before
+    public void setup() throws Exception {
+        cliClientService = new CliClientServiceImpl();
+        cliClientService.init(new CliOptions());
+        this.dataPath = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.dataPath));
+        assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        final List peers = TestUtils.generatePeers(3);
+
+        cluster = new TestCluster(groupId, dataPath, peers);
+        for (final PeerId peer : peers) {
+            cluster.start(peer.getEndpoint());
+        }
+        cluster.waitLeader();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        cliClientService.shutdown();
+        cluster.stopAll();
+        if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+            Thread.sleep(1000);
+            assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        }
+        FileUtils.deleteDirectory(new File(this.dataPath));
+        NodeManager.getInstance().clear();
+        RouteTable.getInstance().reset();
+    }
+
+    @Test
+    public void testUpdateConfSelectLeader() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        assertNull(rt.getConfiguration(groupId));
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        assertEquals(rt.getConfiguration(groupId), new Configuration(cluster.getPeers()));
+        assertNull(rt.selectLeader(groupId));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+        final PeerId leader = rt.selectLeader(groupId);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testUpdateLeaderNull() throws Exception {
+        this.testUpdateConfSelectLeader();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateLeader(groupId, (PeerId) null);
+        assertNull(rt.selectLeader(groupId));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+        final PeerId leader = rt.selectLeader(groupId);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testRefreshLeaderWhenLeaderStops() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        testUpdateConfSelectLeader();
+        PeerId leader = rt.selectLeader(groupId);
+        this.cluster.stop(leader.getEndpoint());
+        this.cluster.waitLeader();
+        final PeerId oldLeader = leader.copy();
+
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+        leader = rt.selectLeader(groupId);
+        assertNotEquals(leader, oldLeader);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testRefreshLeaderWhenFirstPeerDown() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+        cluster.stop(cluster.getPeers().get(0).getEndpoint());
+        Thread.sleep(1000);
+        this.cluster.waitLeader();
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+    }
+
+    @Test
+    public void testRefreshFail() throws Exception {
+        cluster.stopAll();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        final Status status = rt.refreshLeader(cliClientService, groupId, 5000);
+        assertFalse(status.isOk());
+        assertTrue(status.getErrorMsg().contains("Fail to init channel"));
+    }
+
+    @Test
+    public void testRefreshConfiguration() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        final List partConf = new ArrayList<>();
+        partConf.add(cluster.getLeader().getLeaderId());
+        // part of peers conf, only contains leader peer
+        rt.updateConfiguration(groupId, new Configuration(partConf));
+        // fetch all conf
+        final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+        assertTrue(st.isOk());
+        final Configuration newCnf = rt.getConfiguration(groupId);
+        assertArrayEquals(new HashSet<>(cluster.getPeers()).toArray(), new HashSet<>(newCnf.getPeerSet()).toArray());
+    }
+
+    @Test
+    public void testRefreshConfigurationFail() throws Exception {
+        cluster.stopAll();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+        assertFalse(st.isOk());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/StatusTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/StatusTest.java
new file mode 100644
index 0000000..2c3a1e9
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/StatusTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.error.RaftError;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class StatusTest {
+
+    @Test
+    public void testOKStatus() {
+        Status s = new Status();
+        assertTrue(s.isOk());
+        assertEquals(0, s.getCode());
+        assertNull(s.getErrorMsg());
+    }
+
+    @Test
+    public void testStatusOK() {
+        Status s = Status.OK();
+        assertTrue(s.isOk());
+        assertEquals(0, s.getCode());
+        assertNull(s.getErrorMsg());
+        assertNotSame(Status.OK(), s);
+    }
+
+    @Test
+    public void testNewStatus() {
+        Status s = new Status(-2, "test");
+        assertEquals(-2, s.getCode());
+        assertEquals("test", s.getErrorMsg());
+        assertFalse(s.isOk());
+    }
+
+    @Test
+    public void testNewStatusVaridicArgs() {
+        Status s = new Status(-2, "test %s %d", "world", 100);
+        assertEquals(-2, s.getCode());
+        assertEquals("test world 100", s.getErrorMsg());
+        assertFalse(s.isOk());
+    }
+
+    @Test
+    public void testNewStatusRaftError() {
+        Status s = new Status(RaftError.EACCES, "test %s %d", "world", 100);
+        assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+        assertEquals(RaftError.EACCES, s.getRaftError());
+        assertEquals("test world 100", s.getErrorMsg());
+        assertFalse(s.isOk());
+    }
+
+    @Test
+    public void testSetErrorRaftError() {
+        Status s = new Status();
+        s.setError(RaftError.EACCES, "test %s %d", "world", 100);
+        assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+        assertEquals(RaftError.EACCES, s.getRaftError());
+        assertEquals("test world 100", s.getErrorMsg());
+        assertFalse(s.isOk());
+    }
+
+    @Test
+    public void testSetError() {
+        Status s = new Status();
+        s.setError(RaftError.EACCES.getNumber(), "test %s %d", "world", 100);
+        assertEquals(RaftError.EACCES.getNumber(), s.getCode());
+        assertEquals(RaftError.EACCES, s.getRaftError());
+        assertEquals("test world 100", s.getErrorMsg());
+        assertFalse(s.isOk());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java
new file mode 100644
index 0000000..2512e6a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/ClosureQueueTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.closure;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Closure;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ClosureQueueTest {
+    private ClosureQueueImpl queue;
+
+    @Before
+    public void setup() {
+        this.queue = new ClosureQueueImpl();
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private Closure mockClosure(final CountDownLatch latch) {
+        return status -> {
+            if (latch != null) {
+                latch.countDown();
+            }
+        };
+    }
+
+    @Test
+    public void testAppendPop() {
+        for (int i = 0; i < 10; i++) {
+            this.queue.appendPendingClosure(mockClosure(null));
+        }
+        assertEquals(0, this.queue.getFirstIndex());
+        List closures = new ArrayList<>();
+        assertEquals(0, this.queue.popClosureUntil(4, closures));
+        assertEquals(5, closures.size());
+
+        assertEquals(5, this.queue.getFirstIndex());
+
+        closures.clear();
+        assertEquals(5, this.queue.popClosureUntil(4, closures));
+        assertTrue(closures.isEmpty());
+        assertEquals(4, this.queue.popClosureUntil(3, closures));
+        assertTrue(closures.isEmpty());
+
+        assertEquals(-1, this.queue.popClosureUntil(10, closures));
+        assertTrue(closures.isEmpty());
+
+        //pop remaining 5 elements
+        assertEquals(5, this.queue.popClosureUntil(9, closures));
+        assertEquals(5, closures.size());
+        assertEquals(10, this.queue.getFirstIndex());
+        closures.clear();
+        assertEquals(2, this.queue.popClosureUntil(1, closures));
+        assertTrue(closures.isEmpty());
+        assertEquals(4, this.queue.popClosureUntil(3, closures));
+        assertTrue(closures.isEmpty());
+
+        for (int i = 0; i < 10; i++) {
+            this.queue.appendPendingClosure(mockClosure(null));
+        }
+
+        assertEquals(10, this.queue.popClosureUntil(15, closures));
+        assertEquals(6, closures.size());
+        assertEquals(16, this.queue.getFirstIndex());
+
+        assertEquals(-1, this.queue.popClosureUntil(20, closures));
+        assertTrue(closures.isEmpty());
+        assertEquals(16, this.queue.popClosureUntil(19, closures));
+        assertEquals(4, closures.size());
+        assertEquals(20, this.queue.getFirstIndex());
+    }
+
+    @Test
+    public void testResetFirstIndex() {
+        assertEquals(0, this.queue.getFirstIndex());
+        this.queue.resetFirstIndex(10);
+        assertEquals(10, this.queue.getFirstIndex());
+        for (int i = 0; i < 10; i++) {
+            this.queue.appendPendingClosure(mockClosure(null));
+        }
+
+        List closures = new ArrayList<>();
+        assertEquals(5, this.queue.popClosureUntil(4, closures));
+        assertTrue(closures.isEmpty());
+        assertEquals(4, this.queue.popClosureUntil(3, closures));
+        assertTrue(closures.isEmpty());
+
+        assertEquals(10, this.queue.popClosureUntil(19, closures));
+        assertEquals(20, this.queue.getFirstIndex());
+        assertEquals(10, closures.size());
+        // empty ,return index+1
+        assertEquals(21, this.queue.popClosureUntil(20, closures));
+        assertTrue(closures.isEmpty());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java
new file mode 100644
index 0000000..94744bf
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/closure/SynchronizedClosureTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.closure;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Status;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class SynchronizedClosureTest {
+    private SynchronizedClosure done;
+
+    @Before
+    public void setup() {
+        this.done = new SynchronizedClosure(1);
+    }
+
+    @Test
+    public void testAwaitRun() throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        AtomicLong cost = new AtomicLong(0);
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    long start = System.currentTimeMillis();
+                    done.await();
+                    cost.set(System.currentTimeMillis() - start);
+                } catch (InterruptedException e) {
+                    e.printStackTrace();
+                }
+                latch.countDown();
+            }
+        }.start();
+
+        int n = 1000;
+        Thread.sleep(n);
+        this.done.run(Status.OK());
+        latch.await();
+        assertEquals(n, cost.get(), 50);
+        assertTrue(this.done.getStatus().isOk());
+    }
+
+    @Test
+    public void testReset() throws Exception {
+        testAwaitRun();
+        this.done.await();
+        assertTrue(true);
+        this.done.reset();
+        assertNull(this.done.getStatus());
+        testAwaitRun();
+        assertTrue(this.done.getStatus().isOk());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java
new file mode 100644
index 0000000..009933c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationEntryTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+public class ConfigurationEntryTest {
+    @Test
+    public void testStuffMethods() {
+        ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", null);
+        assertTrue(entry.isStable());
+        assertFalse(entry.isEmpty());
+        assertTrue(entry.contains(new PeerId("localhost", 8081)));
+        assertTrue(entry.contains(new PeerId("localhost", 8082)));
+        assertTrue(entry.contains(new PeerId("localhost", 8083)));
+        assertEquals(
+            entry.listPeers(),
+            new HashSet<>(Arrays.asList(new PeerId("localhost", 8081), new PeerId("localhost", 8082), new PeerId(
+                "localhost", 8083))));
+
+    }
+
+    @Test
+    public void testStuffMethodsWithPriority() {
+        ConfigurationEntry entry = TestUtils.getConfEntry(
+            "localhost:8081::100,localhost:8082::100,localhost:8083::100", null);
+        assertTrue(entry.isStable());
+        assertFalse(entry.isEmpty());
+        assertTrue(entry.contains(new PeerId("localhost", 8081, 0, 100)));
+        assertTrue(entry.contains(new PeerId("localhost", 8082, 0, 100)));
+        assertTrue(entry.contains(new PeerId("localhost", 8083, 0, 100)));
+        assertEquals(
+            entry.listPeers(),
+            new HashSet<>(Arrays.asList(new PeerId("localhost", 8081, 0, 100), new PeerId("localhost", 8082, 0, 100),
+                new PeerId("localhost", 8083, 0, 100))));
+
+    }
+
+    @Test
+    public void testIsValid() {
+        ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", null);
+        assertTrue(entry.isValid());
+
+        entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083",
+            "localhost:8081,localhost:8082,localhost:8084");
+        assertTrue(entry.isValid());
+
+        entry.getConf().addLearner(new PeerId("localhost", 8084));
+        assertFalse(entry.isValid());
+        entry.getConf().addLearner(new PeerId("localhost", 8081));
+        assertFalse(entry.isValid());
+    }
+
+    @Test
+    public void testIsStable() {
+        ConfigurationEntry entry = TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083",
+            "localhost:8080,localhost:8081,localhost:8082");
+        assertFalse(entry.isStable());
+        assertEquals(4, entry.listPeers().size());
+        assertTrue(entry.contains(new PeerId("localhost", 8080)));
+        assertTrue(entry.contains(new PeerId("localhost", 8081)));
+        assertTrue(entry.contains(new PeerId("localhost", 8082)));
+        assertTrue(entry.contains(new PeerId("localhost", 8083)));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java
new file mode 100644
index 0000000..d3771c6
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationManagerTest.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class ConfigurationManagerTest {
+
+    private ConfigurationManager confManager;
+
+    @Before
+    public void setup() {
+        this.confManager = new ConfigurationManager();
+    }
+
+    @Test
+    public void testGetStuff() {
+        ConfigurationEntry lastConf = this.confManager.getLastConfiguration();
+        ConfigurationEntry snapshot = this.confManager.getSnapshot();
+        assertSame(snapshot, lastConf);
+        assertSame(snapshot, this.confManager.get(0));
+
+        ConfigurationEntry confEntry1 = TestUtils.getConfEntry("localhost:8080", null);
+        confEntry1.setId(new LogId(0, 0));
+        assertTrue(this.confManager.add(confEntry1));
+        lastConf = this.confManager.getLastConfiguration();
+        assertNotSame(snapshot, lastConf);
+        assertSame(confEntry1, lastConf);
+
+        assertSame(confEntry1, this.confManager.get(0));
+        assertSame(confEntry1, this.confManager.get(1));
+        assertSame(confEntry1, this.confManager.get(2));
+
+        ConfigurationEntry confEntry2 = TestUtils.getConfEntry("localhost:8080,localhost:8081", "localhost:8080");
+        confEntry2.setId(new LogId(1, 1));
+        assertTrue(this.confManager.add(confEntry2));
+
+        lastConf = this.confManager.getLastConfiguration();
+        assertNotSame(snapshot, lastConf);
+        assertSame(confEntry2, lastConf);
+
+        assertSame(confEntry1, this.confManager.get(0));
+        assertSame(confEntry2, this.confManager.get(1));
+        assertSame(confEntry2, this.confManager.get(2));
+
+        ConfigurationEntry confEntry3 = TestUtils.getConfEntry("localhost:8080,localhost:8081,localhost:8082",
+            "localhost:8080,localhost:8081");
+        confEntry3.setId(new LogId(2, 1));
+        assertTrue(this.confManager.add(confEntry3));
+
+        lastConf = this.confManager.getLastConfiguration();
+        assertNotSame(snapshot, lastConf);
+        assertSame(confEntry3, lastConf);
+
+        assertSame(confEntry1, this.confManager.get(0));
+        assertSame(confEntry2, this.confManager.get(1));
+        assertSame(confEntry3, this.confManager.get(2));
+    }
+
+    private ConfigurationEntry createEnry(int index) {
+        ConfigurationEntry configurationEntry = new ConfigurationEntry();
+        configurationEntry.getId().setIndex(index);
+        return configurationEntry;
+    }
+
+    @Test
+    public void testTruncate() {
+        for (int i = 0; i < 10; i++) {
+            assertTrue(this.confManager.add(createEnry(i)));
+        }
+
+        assertEquals(9, this.confManager.getLastConfiguration().getId().getIndex());
+        assertEquals(5, this.confManager.get(5).getId().getIndex());
+        assertEquals(6, this.confManager.get(6).getId().getIndex());
+        this.confManager.truncatePrefix(6);
+        //truncated, so is snapshot index
+        assertEquals(0, this.confManager.get(5).getId().getIndex());
+        assertEquals(6, this.confManager.get(6).getId().getIndex());
+        assertEquals(9, this.confManager.getLastConfiguration().getId().getIndex());
+
+        this.confManager.truncateSuffix(7);
+        assertEquals(0, this.confManager.get(5).getId().getIndex());
+        assertEquals(6, this.confManager.get(6).getId().getIndex());
+        assertEquals(7, this.confManager.getLastConfiguration().getId().getIndex());
+        assertEquals(7, this.confManager.get(9).getId().getIndex());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java
new file mode 100644
index 0000000..cb48fce
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/conf/ConfigurationTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.conf;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.entity.PeerId;
+
+public class ConfigurationTest {
+
+    @Test
+    public void testToStringParseStuff() {
+        final String confStr = "localhost:8081,localhost:8082,localhost:8083";
+        final Configuration conf = JRaftUtils.getConfiguration(confStr);
+        assertEquals(3, conf.size());
+        for (final PeerId peer : conf) {
+            assertTrue(peer.toString().startsWith("localhost:80"));
+        }
+        assertFalse(conf.isEmpty());
+        assertEquals(confStr, conf.toString());
+        final Configuration newConf = new Configuration();
+        assertTrue(newConf.parse(conf.toString()));
+        assertEquals(3, newConf.getPeerSet().size());
+        assertTrue(newConf.contains(new PeerId("localhost", 8081)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8082)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8083)));
+        assertEquals(confStr, newConf.toString());
+        assertEquals(conf.hashCode(), newConf.hashCode());
+        assertEquals(conf, newConf);
+    }
+
+    @Test
+    public void testToStringParseStuffWithPriority() {
+        final String confStr = "localhost:8081:1:100,localhost:8082:1:100,localhost:8083:1:100";
+        final Configuration conf = JRaftUtils.getConfiguration(confStr);
+        assertEquals(3, conf.size());
+        for (final PeerId peer : conf) {
+            assertTrue(peer.toString().startsWith("localhost:80"));
+            assertEquals(100, peer.getPriority());
+            assertEquals(1, peer.getIdx());
+        }
+        assertFalse(conf.isEmpty());
+        assertEquals(confStr, conf.toString());
+        final Configuration newConf = new Configuration();
+        assertTrue(newConf.parse(conf.toString()));
+        assertEquals(3, newConf.getPeerSet().size());
+        assertTrue(newConf.contains(new PeerId("localhost", 8081, 1, 100)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8082, 1, 100)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8083, 1, 100)));
+        assertEquals(confStr, newConf.toString());
+        assertEquals(conf.hashCode(), newConf.hashCode());
+        assertEquals(conf, newConf);
+    }
+
+    @Test
+    public void testToStringParseStuffWithPriorityAndNone() {
+        final String confStr = "localhost:8081,localhost:8082,localhost:8083:1:100";
+        final Configuration conf = JRaftUtils.getConfiguration(confStr);
+        assertEquals(3, conf.size());
+        for (final PeerId peer : conf) {
+            assertTrue(peer.toString().startsWith("localhost:80"));
+
+            if (peer.getIp().equals("localhost:8083")) {
+                assertEquals(100, peer.getPriority());
+                assertEquals(1, peer.getIdx());
+            }
+        }
+        assertFalse(conf.isEmpty());
+        assertEquals(confStr, conf.toString());
+        final Configuration newConf = new Configuration();
+        assertTrue(newConf.parse(conf.toString()));
+        assertEquals(3, newConf.getPeerSet().size());
+        assertTrue(newConf.contains(new PeerId("localhost", 8081)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8082)));
+        assertTrue(newConf.contains(new PeerId("localhost", 8083, 1, 100)));
+        assertEquals(confStr, newConf.toString());
+        assertEquals(conf.hashCode(), newConf.hashCode());
+        assertEquals(conf, newConf);
+    }
+
+    @Test
+    public void testLearnerStuff() {
+        final String confStr = "localhost:8081,localhost:8082,localhost:8083";
+        final Configuration conf = JRaftUtils.getConfiguration(confStr);
+        assertEquals(3, conf.size());
+        assertEquals(confStr, conf.toString());
+        assertTrue(conf.isValid());
+
+        PeerId learner1 = new PeerId("192.168.1.1", 8081);
+        assertTrue(conf.addLearner(learner1));
+        assertFalse(conf.addLearner(learner1));
+        PeerId learner2 = new PeerId("192.168.1.2", 8081);
+        assertTrue(conf.addLearner(learner2));
+
+        assertEquals(2, conf.getLearners().size());
+        assertTrue(conf.getLearners().contains(learner1));
+        assertTrue(conf.getLearners().contains(learner2));
+
+        String newConfStr = "localhost:8081,localhost:8082,localhost:8083,192.168.1.1:8081/learner,192.168.1.2:8081/learner";
+        assertEquals(newConfStr, conf.toString());
+        assertTrue(conf.isValid());
+
+        final Configuration newConf = JRaftUtils.getConfiguration(newConfStr);
+        assertEquals(newConf, conf);
+        assertEquals(2, newConf.getLearners().size());
+        assertEquals(newConfStr, newConf.toString());
+        assertTrue(newConf.isValid());
+
+        // Also adds localhost:8081 as learner
+        assertTrue(conf.addLearner(new PeerId("localhost", 8081)));
+        // The conf is invalid, because the peers and learns have intersection.
+        assertFalse(conf.isValid());
+    }
+
+    @Test
+    public void testCopy() {
+        final Configuration conf = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+        final Configuration copied = conf.copy();
+        assertEquals(conf, copied);
+        assertNotSame(conf, copied);
+        assertEquals(copied.size(), 3);
+        assertEquals("localhost:8081,localhost:8082,localhost:8083", copied.toString());
+
+        final PeerId newPeer = new PeerId("localhost", 8084);
+        conf.addPeer(newPeer);
+        assertEquals(copied.size(), 3);
+        assertEquals(conf.size(), 4);
+        assertTrue(conf.contains(newPeer));
+        assertFalse(copied.contains(newPeer));
+    }
+
+    @Test
+    public void testReset() {
+        final Configuration conf = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+        assertFalse(conf.isEmpty());
+        conf.reset();
+        assertTrue(conf.isEmpty());
+        assertTrue(conf.getPeerSet().isEmpty());
+    }
+
+    @Test
+    public void testDiff() {
+        final Configuration conf1 = JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083");
+        final Configuration conf2 = JRaftUtils
+            .getConfiguration("localhost:8081,localhost:8083,localhost:8085,localhost:8086");
+        final Configuration included = new Configuration();
+        final Configuration excluded = new Configuration();
+        conf1.diff(conf2, included, excluded);
+        assertEquals("localhost:8082", included.toString());
+        assertEquals("localhost:8085,localhost:8086", excluded.toString());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java
new file mode 100644
index 0000000..a310e56
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/BallotBoxTest.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ClosureQueueImpl;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.BallotBoxOptions;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class BallotBoxTest {
+    private BallotBox        box;
+    @Mock
+    private FSMCaller        waiter;
+    private ClosureQueueImpl closureQueue;
+
+    @Before
+    public void setup() {
+        BallotBoxOptions opts = new BallotBoxOptions();
+        this.closureQueue = new ClosureQueueImpl();
+        opts.setClosureQueue(this.closureQueue);
+        opts.setWaiter(this.waiter);
+        box = new BallotBox();
+        assertTrue(box.init(opts));
+    }
+
+    @After
+    public void teardown() {
+        box.shutdown();
+    }
+
+    @Test
+    public void testResetPendingIndex() {
+        assertEquals(0, closureQueue.getFirstIndex());
+        assertEquals(0, box.getPendingIndex());
+        assertTrue(box.resetPendingIndex(1));
+        assertEquals(1, closureQueue.getFirstIndex());
+        assertEquals(1, box.getPendingIndex());
+    }
+
+    @Test
+    public void testAppendPendingTask() {
+        assertTrue(this.box.getPendingMetaQueue().isEmpty());
+        assertTrue(this.closureQueue.getQueue().isEmpty());
+        assertFalse(this.box.appendPendingTask(
+            JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+            JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+                @Override
+                public void run(Status status) {
+
+                }
+            }));
+        assertTrue(box.resetPendingIndex(1));
+        assertTrue(this.box.appendPendingTask(
+            JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+            JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+                @Override
+                public void run(Status status) {
+
+                }
+            }));
+
+        assertEquals(1, this.box.getPendingMetaQueue().size());
+        assertEquals(1, this.closureQueue.getQueue().size());
+    }
+
+    @Test
+    public void testClearPendingTasks() {
+        testAppendPendingTask();
+        this.box.clearPendingTasks();
+        assertTrue(this.box.getPendingMetaQueue().isEmpty());
+        assertTrue(this.closureQueue.getQueue().isEmpty());
+        assertEquals(0, closureQueue.getFirstIndex());
+    }
+
+    @Test
+    public void testCommitAt() {
+        assertFalse(this.box.commitAt(1, 3, new PeerId("localhost", 8081)));
+        assertTrue(box.resetPendingIndex(1));
+        assertTrue(this.box.appendPendingTask(
+            JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"),
+            JRaftUtils.getConfiguration("localhost:8081"), new Closure() {
+
+                @Override
+                public void run(Status status) {
+
+                }
+            }));
+        assertEquals(0, this.box.getLastCommittedIndex());
+        try {
+            this.box.commitAt(1, 3, new PeerId("localhost", 8081));
+            fail();
+        } catch (ArrayIndexOutOfBoundsException e) {
+
+        }
+        assertTrue(this.box.commitAt(1, 1, new PeerId("localhost", 8081)));
+        assertEquals(0, this.box.getLastCommittedIndex());
+        assertEquals(1, this.box.getPendingIndex());
+        assertTrue(this.box.commitAt(1, 1, new PeerId("localhost", 8082)));
+        assertEquals(1, this.box.getLastCommittedIndex());
+        assertEquals(2, this.box.getPendingIndex());
+        Mockito.verify(this.waiter, Mockito.only()).onCommitted(1);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetLastCommittedIndexHasPending() {
+        assertTrue(box.resetPendingIndex(1));
+        assertFalse(this.box.setLastCommittedIndex(1));
+    }
+
+    @Test
+    public void testSetLastCommittedIndexLessThan() {
+        assertFalse(this.box.setLastCommittedIndex(-1));
+    }
+
+    @Test
+    public void testSetLastCommittedIndex() {
+        assertEquals(0, this.box.getLastCommittedIndex());
+        assertTrue(this.box.setLastCommittedIndex(1));
+        assertEquals(1, this.box.getLastCommittedIndex());
+        Mockito.verify(this.waiter, Mockito.only()).onCommitted(1);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
new file mode 100644
index 0000000..c9a7d14
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.CliService;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CliServiceTest {
+
+    private String           dataPath;
+
+    private TestCluster      cluster;
+    private final String     groupId           = "CliServiceTest";
+
+    private CliService       cliService;
+
+    private Configuration    conf;
+
+    @Rule
+    public TestName          testName          = new TestName();
+
+    private static final int LEARNER_PORT_STEP = 100;
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+        this.dataPath = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.dataPath));
+        assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        final List peers = TestUtils.generatePeers(3);
+
+        final LinkedHashSet learners = new LinkedHashSet<>();
+        //2 learners
+        for (int i = 0; i < 2; i++) {
+            learners.add(new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + i));
+        }
+
+        this.cluster = new TestCluster(this.groupId, this.dataPath, peers, learners, 300);
+        for (final PeerId peer : peers) {
+            this.cluster.start(peer.getEndpoint());
+        }
+
+        for (final PeerId peer : learners) {
+            this.cluster.startLearner(peer);
+        }
+
+        this.cluster.waitLeader();
+
+        this.cliService = new CliServiceImpl();
+        this.conf = new Configuration(peers, learners);
+        assertTrue(this.cliService.init(new CliOptions()));
+    }
+
+    @After
+    public void teardown() throws Exception {
+        this.cliService.shutdown();
+        this.cluster.stopAll();
+        if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+            Thread.sleep(1000);
+            assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        }
+        FileUtils.deleteDirectory(new File(this.dataPath));
+        NodeManager.getInstance().clear();
+        RouteTable.getInstance().reset();
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+    }
+
+    @Test
+    public void testTransferLeader() throws Exception {
+        final PeerId leader = this.cluster.getLeader().getNodeId().getPeerId().copy();
+        assertNotNull(leader);
+
+        final Set peers = this.conf.getPeerSet();
+        PeerId targetPeer = null;
+        for (final PeerId peer : peers) {
+            if (!peer.equals(leader)) {
+                targetPeer = peer;
+                break;
+            }
+        }
+        assertNotNull(targetPeer);
+        assertTrue(this.cliService.transferLeader(this.groupId, this.conf, targetPeer).isOk());
+        this.cluster.waitLeader();
+        assertEquals(targetPeer, this.cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private void sendTestTaskAndWait(final Node node, final int code) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(10);
+        for (int i = 0; i < 10; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(code, null, latch));
+            node.apply(task);
+        }
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testLearnerServices() throws Exception {
+        final PeerId learner3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + 3);
+        assertTrue(this.cluster.startLearner(learner3));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(500);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+                assertEquals(10, fsm.getLogs().size());
+            }
+        }
+        assertEquals(0, this.cluster.getFsmByPeer(learner3).getLogs().size());
+        List oldLearners = new ArrayList(this.conf.getLearners());
+        assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Add learner3
+        this.cliService.addLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        Thread.sleep(1000);
+        assertEquals(10, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(1000);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+
+        }
+        List newLearners = new ArrayList<>(oldLearners);
+        newLearners.add(learner3);
+        assertEquals(newLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(newLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Remove  3
+        this.cliService.removeLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(1000);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+        // Latest 10 logs are not replicated to learner3, because it's removed.
+        assertEquals(20, this.cluster.getFsmByPeer(learner3).getLogs().size());
+        assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Set learners into [learner3]
+        this.cliService.resetLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        Thread.sleep(100);
+        assertEquals(30, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(1000);
+        // Latest 10 logs are not replicated to learner1 and learner2, because they were removed by resetting learners set.
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!oldLearners.contains(new PeerId(fsm.getAddress(), 0))) {
+                assertEquals(40, fsm.getLogs().size());
+            } else {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+        assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(Arrays.asList(learner3), this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Stop learner3
+        this.cluster.stop(learner3.getEndpoint());
+        Thread.sleep(1000);
+        assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+        assertTrue(this.cliService.getAliveLearners(this.groupId, this.conf).isEmpty());
+        final PeerId learner4 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + 4);
+        assertTrue(this.cluster.startLearner(learner4));
+        this.cliService.addLearners(this.groupId, this.conf, Arrays.asList(learner4));
+        Thread.sleep(1000);
+        assertTrue(this.cliService.getAliveLearners(this.groupId, this.conf).size() == 1);
+        assertTrue(this.cliService.learner2Follower(this.groupId, this.conf, learner4).isOk());
+        Thread.sleep(1000);
+        List currentLearners = this.cliService.getAliveLearners(this.groupId, this.conf);
+        assertFalse(currentLearners.contains(learner4));
+        List currentFollowers = this.cliService.getPeers(this.groupId, this.conf);
+        assertTrue(currentFollowers.contains(learner4));
+        this.cluster.stop(learner4.getEndpoint());
+    }
+
+    @Test
+    public void testAddPeerRemovePeer() throws Exception {
+        final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+        assertTrue(this.cluster.start(peer3.getEndpoint()));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(100);
+        assertEquals(0, this.cluster.getFsmByPeer(peer3).getLogs().size());
+
+        assertTrue(this.cliService.addPeer(this.groupId, this.conf, peer3).isOk());
+        Thread.sleep(100);
+        assertEquals(10, this.cluster.getFsmByPeer(peer3).getLogs().size());
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(100);
+        assertEquals(6, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+
+        //remove peer3
+        assertTrue(this.cliService.removePeer(this.groupId, this.conf, peer3).isOk());
+        Thread.sleep(200);
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(1000);
+        assertEquals(6, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (fsm.getAddress().equals(peer3.getEndpoint())) {
+                assertEquals(20, fsm.getLogs().size());
+            } else {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+    }
+
+    @Test
+    public void testChangePeers() throws Exception {
+        final List newPeers = TestUtils.generatePeers(10);
+        newPeers.removeAll(this.conf.getPeerSet());
+        for (final PeerId peer : newPeers) {
+            assertTrue(this.cluster.start(peer.getEndpoint()));
+        }
+        this.cluster.waitLeader();
+        final Node oldLeaderNode = this.cluster.getLeader();
+        assertNotNull(oldLeaderNode);
+        final PeerId oldLeader = oldLeaderNode.getNodeId().getPeerId();
+        assertNotNull(oldLeader);
+        assertTrue(this.cliService.changePeers(this.groupId, this.conf, new Configuration(newPeers)).isOk());
+        this.cluster.waitLeader();
+        final PeerId newLeader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotEquals(oldLeader, newLeader);
+        assertTrue(newPeers.contains(newLeader));
+    }
+
+    @Test
+    public void testSnapshot() throws Exception {
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        assertEquals(5, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(0, fsm.getSaveSnapshotTimes());
+        }
+
+        for (final PeerId peer : this.conf) {
+            assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+        }
+        for (final PeerId peer : this.conf.getLearners()) {
+            assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+        }
+        Thread.sleep(1000);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(1, fsm.getSaveSnapshotTimes());
+        }
+    }
+
+    @Test
+    public void testGetPeers() throws Exception {
+        PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+        // stop one peer
+        final List peers = this.conf.getPeers();
+        this.cluster.stop(peers.get(0).getEndpoint());
+
+        this.cluster.waitLeader();
+
+        leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+        this.cluster.stopAll();
+
+        try {
+            this.cliService.getPeers(this.groupId, this.conf);
+            fail();
+        } catch (final IllegalStateException e) {
+            assertEquals("Fail to get leader of group " + this.groupId, e.getMessage());
+        }
+    }
+
+    @Test
+    public void testGetAlivePeers() throws Exception {
+        PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+        // stop one peer
+        final List peers = this.conf.getPeers();
+        this.cluster.stop(peers.get(0).getEndpoint());
+        peers.remove(0);
+
+        this.cluster.waitLeader();
+
+        Thread.sleep(1000);
+
+        leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(new HashSet<>(peers).toArray(),
+            new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+        this.cluster.stopAll();
+
+        try {
+            this.cliService.getAlivePeers(this.groupId, this.conf);
+            fail();
+        } catch (final IllegalStateException e) {
+            assertEquals("Fail to get leader of group " + this.groupId, e.getMessage());
+        }
+    }
+
+    @Test
+    public void testRebalance() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        groupIds.add("group_5");
+        groupIds.add("group_6");
+        groupIds.add("group_7");
+        groupIds.add("group_8");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockCliService(rebalancedLeaderIds, new PeerId("host_1", 8080));
+
+        assertTrue(cliService.rebalance(groupIds, conf, rebalancedLeaderIds).isOk());
+        assertEquals(groupIds.size(), rebalancedLeaderIds.size());
+
+        final Map ret = new HashMap<>();
+        for (Map.Entry entry : rebalancedLeaderIds.entrySet()) {
+            ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+        }
+        final int expectedAvgLeaderNum = (int) Math.ceil((double) groupIds.size() / conf.size());
+        for (Map.Entry entry : ret.entrySet()) {
+            System.out.println(entry);
+            assertTrue(entry.getValue() <= expectedAvgLeaderNum);
+        }
+    }
+
+    @Test
+    public void testRebalanceOnLeaderFail() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockLeaderFailCliService();
+
+        assertEquals("Fail to get leader", cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+    }
+
+    @Test
+    public void testRelalanceOnTransferLeaderFail() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        groupIds.add("group_5");
+        groupIds.add("group_6");
+        groupIds.add("group_7");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockTransferLeaderFailCliService(rebalancedLeaderIds,
+            new PeerId("host_1", 8080));
+
+        assertEquals("Fail to transfer leader",
+            cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+        assertTrue(groupIds.size() >= rebalancedLeaderIds.size());
+
+        final Map ret = new HashMap<>();
+        for (Map.Entry entry : rebalancedLeaderIds.entrySet()) {
+            ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+        }
+        for (Map.Entry entry : ret.entrySet()) {
+            System.out.println(entry);
+            assertEquals(new PeerId("host_1", 8080), entry.getKey());
+        }
+    }
+
+    class MockCliService extends CliServiceImpl {
+
+        private final Map rebalancedLeaderIds;
+        private final PeerId              initialLeaderId;
+
+        MockCliService(final Map rebalancedLeaderIds, final PeerId initialLeaderId) {
+            this.rebalancedLeaderIds = rebalancedLeaderIds;
+            this.initialLeaderId = initialLeaderId;
+        }
+
+        @Override
+        public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+            final PeerId ret = this.rebalancedLeaderIds.get(groupId);
+            if (ret != null) {
+                leaderId.parse(ret.toString());
+            } else {
+                leaderId.parse(this.initialLeaderId.toString());
+            }
+            return Status.OK();
+        }
+
+        @Override
+        public List getAlivePeers(final String groupId, final Configuration conf) {
+            return conf.getPeers();
+        }
+
+        @Override
+        public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+            return Status.OK();
+        }
+    }
+
+    class MockLeaderFailCliService extends MockCliService {
+
+        MockLeaderFailCliService() {
+            super(null, null);
+        }
+
+        @Override
+        public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+            return new Status(-1, "Fail to get leader");
+        }
+    }
+
+    class MockTransferLeaderFailCliService extends MockCliService {
+
+        MockTransferLeaderFailCliService(final Map rebalancedLeaderIds, final PeerId initialLeaderId) {
+            super(rebalancedLeaderIds, initialLeaderId);
+        }
+
+        @Override
+        public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+            return new Status(-1, "Fail to transfer leader");
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
new file mode 100644
index 0000000..3e25838
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import static org.junit.Assert.assertEquals;
+
+public class ExpectClosure implements Closure {
+    private final int            expectedErrCode;
+    private final String         expectErrMsg;
+    private final CountDownLatch latch;
+    private AtomicInteger        successCount;
+
+    public ExpectClosure(final CountDownLatch latch) {
+        this(RaftError.SUCCESS, latch);
+    }
+
+    public ExpectClosure(final RaftError expectedErrCode, final CountDownLatch latch) {
+        this(expectedErrCode, null, latch);
+
+    }
+
+    public ExpectClosure(final RaftError expectedErrCode, final String expectErrMsg, final CountDownLatch latch) {
+        super();
+        this.expectedErrCode = expectedErrCode.getNumber();
+        this.expectErrMsg = expectErrMsg;
+        this.latch = latch;
+    }
+
+    public ExpectClosure(final int code, final String expectErrMsg, final CountDownLatch latch) {
+        this(code, expectErrMsg, latch, null);
+    }
+
+    public ExpectClosure(final int code, final String expectErrMsg, final CountDownLatch latch,
+                         final AtomicInteger successCount) {
+        super();
+        this.expectedErrCode = code;
+        this.expectErrMsg = expectErrMsg;
+        this.latch = latch;
+        this.successCount = successCount;
+    }
+
+    @Override
+    public void run(final Status status) {
+        if (this.expectedErrCode >= 0) {
+            assertEquals(this.expectedErrCode, status.getCode());
+        }
+        if (this.expectErrMsg != null) {
+            assertEquals(this.expectErrMsg, status.getErrorMsg());
+        }
+        if (status.isOk() && this.successCount != null) {
+            this.successCount.incrementAndGet();
+        }
+        this.latch.countDown();
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java
new file mode 100644
index 0000000..da58426
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/FSMCallerTest.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ClosureQueueImpl;
+import com.alipay.sofa.jraft.closure.LoadSnapshotClosure;
+import com.alipay.sofa.jraft.closure.SaveSnapshotClosure;
+import com.alipay.sofa.jraft.entity.EnumOutter.EntryType;
+import com.alipay.sofa.jraft.entity.EnumOutter.ErrorType;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.FSMCallerOptions;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class FSMCallerTest {
+    private FSMCallerImpl    fsmCaller;
+    @Mock
+    private NodeImpl         node;
+    @Mock
+    private StateMachine     fsm;
+    @Mock
+    private LogManager       logManager;
+    private ClosureQueueImpl closureQueue;
+
+    @Before
+    public void setup() {
+        this.fsmCaller = new FSMCallerImpl();
+        this.closureQueue = new ClosureQueueImpl();
+        final FSMCallerOptions opts = new FSMCallerOptions();
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+        opts.setNode(this.node);
+        opts.setFsm(this.fsm);
+        opts.setLogManager(this.logManager);
+        opts.setBootstrapId(new LogId(10, 1));
+        opts.setClosureQueue(this.closureQueue);
+        assertTrue(this.fsmCaller.init(opts));
+    }
+
+    @After
+    public void teardown() throws Exception {
+        if (this.fsmCaller != null) {
+            this.fsmCaller.shutdown();
+            this.fsmCaller.join();
+        }
+    }
+
+    @Test
+    public void testShutdownJoin() throws Exception {
+        this.fsmCaller.shutdown();
+        this.fsmCaller.join();
+        this.fsmCaller = null;
+    }
+
+    @Test
+    public void testOnCommittedError() throws Exception {
+        Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+        Mockito.when(this.logManager.getEntry(11)).thenReturn(null);
+
+        assertTrue(this.fsmCaller.onCommitted(11));
+
+        this.fsmCaller.flush();
+        assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+        Mockito.verify(this.logManager).setAppliedId(new LogId(10, 1));
+        assertFalse(this.fsmCaller.getError().getStatus().isOk());
+        assertEquals("Fail to get entry at index=11 while committed_index=11", this.fsmCaller.getError().getStatus()
+            .getErrorMsg());
+    }
+
+    @Test
+    public void testOnCommitted() throws Exception {
+        final LogEntry log = new LogEntry(EntryType.ENTRY_TYPE_DATA);
+        log.getId().setIndex(11);
+        log.getId().setTerm(1);
+        Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+        Mockito.when(this.logManager.getEntry(11)).thenReturn(log);
+        final ArgumentCaptor itArg = ArgumentCaptor.forClass(Iterator.class);
+
+        assertTrue(this.fsmCaller.onCommitted(11));
+
+        this.fsmCaller.flush();
+        assertEquals(this.fsmCaller.getLastAppliedIndex(), 11);
+        Mockito.verify(this.fsm).onApply(itArg.capture());
+        final Iterator it = itArg.getValue();
+        assertFalse(it.hasNext());
+        assertEquals(it.getIndex(), 12);
+        Mockito.verify(this.logManager).setAppliedId(new LogId(11, 1));
+        assertTrue(this.fsmCaller.getError().getStatus().isOk());
+    }
+
+    @Test
+    public void testOnSnapshotLoad() throws Exception {
+        final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+        final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
+        Mockito.when(reader.load()).thenReturn(meta);
+        Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(true);
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+
+            @Override
+            public SnapshotReader start() {
+                return reader;
+            }
+        });
+        latch.await();
+        assertEquals(this.fsmCaller.getLastAppliedIndex(), 12);
+        Mockito.verify(this.fsm).onConfigurationCommitted(Mockito.any());
+    }
+
+    @Test
+    public void testOnSnapshotLoadFSMError() throws Exception {
+        final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+        final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(12).setLastIncludedTerm(1).build();
+        Mockito.when(reader.load()).thenReturn(meta);
+        Mockito.when(this.fsm.onSnapshotLoad(reader)).thenReturn(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertFalse(status.isOk());
+                assertEquals(-1, status.getCode());
+                assertEquals("StateMachine onSnapshotLoad failed", status.getErrorMsg());
+                latch.countDown();
+            }
+
+            @Override
+            public SnapshotReader start() {
+                return reader;
+            }
+        });
+        latch.await();
+        assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+    }
+
+    @Test
+    public void testOnSnapshotSaveEmptyConf() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.fsmCaller.onSnapshotSave(new SaveSnapshotClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertFalse(status.isOk());
+                assertEquals("Empty conf entry for lastAppliedIndex=10", status.getErrorMsg());
+                latch.countDown();
+            }
+
+            @Override
+            public SnapshotWriter start(final SnapshotMeta meta) {
+                // TODO Auto-generated method stub
+                return null;
+            }
+        });
+        latch.await();
+    }
+
+    @Test
+    public void testOnSnapshotSave() throws Exception {
+        final SnapshotWriter writer = Mockito.mock(SnapshotWriter.class);
+        Mockito.when(this.logManager.getConfiguration(10)).thenReturn(
+            TestUtils.getConfEntry("localhost:8081,localhost:8082,localhost:8083", "localhost:8081"));
+        final SaveSnapshotClosure done = new SaveSnapshotClosure() {
+
+            @Override
+            public void run(final Status status) {
+
+            }
+
+            @Override
+            public SnapshotWriter start(final SnapshotMeta meta) {
+                assertEquals(10, meta.getLastIncludedIndex());
+                return writer;
+            }
+        };
+        this.fsmCaller.onSnapshotSave(done);
+        this.fsmCaller.flush();
+        Mockito.verify(this.fsm).onSnapshotSave(writer, done);
+    }
+
+    @Test
+    public void testOnLeaderStartStop() throws Exception {
+        this.fsmCaller.onLeaderStart(11);
+        this.fsmCaller.flush();
+        Mockito.verify(this.fsm).onLeaderStart(11);
+
+        final Status status = new Status(-1, "test");
+        this.fsmCaller.onLeaderStop(status);
+        this.fsmCaller.flush();
+        Mockito.verify(this.fsm).onLeaderStop(status);
+    }
+
+    @Test
+    public void testOnStartStopFollowing() throws Exception {
+        final LeaderChangeContext ctx = new LeaderChangeContext(null, 11, Status.OK());
+        this.fsmCaller.onStartFollowing(ctx);
+        this.fsmCaller.flush();
+        Mockito.verify(this.fsm).onStartFollowing(ctx);
+
+        this.fsmCaller.onStopFollowing(ctx);
+        this.fsmCaller.flush();
+        Mockito.verify(this.fsm).onStopFollowing(ctx);
+    }
+
+    @Test
+    public void testOnError() throws Exception {
+        this.fsmCaller.onError(new RaftException(ErrorType.ERROR_TYPE_LOG, new Status(-1, "test")));
+        this.fsmCaller.flush();
+        assertFalse(this.fsmCaller.getError().getStatus().isOk());
+        assertEquals(ErrorType.ERROR_TYPE_LOG, this.fsmCaller.getError().getType());
+        Mockito.verify(this.node).onError(Mockito.any());
+        Mockito.verify(this.fsm).onError(Mockito.any());
+    }
+
+    @Test
+    public void testOnSnapshotLoadStale() throws Exception {
+        final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+
+        final SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(5).setLastIncludedTerm(1).build();
+        Mockito.when(reader.load()).thenReturn(meta);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.fsmCaller.onSnapshotLoad(new LoadSnapshotClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertFalse(status.isOk());
+                assertEquals(RaftError.ESTALE, status.getRaftError());
+                latch.countDown();
+            }
+
+            @Override
+            public SnapshotReader start() {
+                return reader;
+            }
+        });
+        latch.await();
+        assertEquals(this.fsmCaller.getLastAppliedIndex(), 10);
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java
new file mode 100644
index 0000000..2de7e32
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorImplTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.LogManager;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class IteratorImplTest {
+
+    private IteratorImpl  iter;
+    @Mock
+    private StateMachine  fsm;
+    @Mock
+    private LogManager    logManager;
+    private List closures;
+    private AtomicLong    applyingIndex;
+
+    @Before
+    public void setup() {
+        this.applyingIndex = new AtomicLong(0);
+        this.closures = new ArrayList<>();
+        for (int i = 0; i < 11; i++) {
+            this.closures.add(new MockClosure());
+            final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+            log.getId().setIndex(i);
+            log.getId().setTerm(1);
+            Mockito.when(this.logManager.getEntry(i)).thenReturn(log);
+        }
+        this.iter = new IteratorImpl(fsm, logManager, closures, 0L, 0L, 10L, applyingIndex);
+    }
+
+    @Test
+    public void testPredicates() {
+        assertTrue(this.iter.isGood());
+        assertFalse(this.iter.hasError());
+    }
+
+    @Test
+    public void testNext() {
+        int i = 1;
+        while (iter.isGood()) {
+            assertEquals(i, iter.getIndex());
+            assertNotNull(iter.done());
+            final LogEntry log = iter.entry();
+            assertEquals(i, log.getId().getIndex());
+            assertEquals(1, log.getId().getTerm());
+            iter.next();
+            i++;
+        }
+        assertEquals(i, 11);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetErrorAndRollbackInvalid() {
+        this.iter.setErrorAndRollback(-1, null);
+    }
+
+    @Test
+    public void testRunTheRestClosureWithError() throws Exception {
+        testSetErrorAndRollback();
+        for (final Closure closure : this.closures) {
+            final MockClosure mc = (MockClosure) closure;
+            assertNull(mc.s);
+        }
+
+        this.iter.runTheRestClosureWithError();
+        Thread.sleep(500);
+        int i = 0;
+        for (final Closure closure : this.closures) {
+            i++;
+            final MockClosure mc = (MockClosure) closure;
+            if (i < 7) {
+                assertNull(mc.s);
+            } else {
+                final Status s = mc.s;
+                Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), s.getCode());
+                assertEquals(
+                    "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+                    s.getErrorMsg());
+            }
+        }
+    }
+
+    @Test
+    public void testSetErrorAndRollback() {
+        testNext();
+        assertFalse(iter.hasError());
+        this.iter.setErrorAndRollback(5, new Status(-1, "test"));
+        assertTrue(iter.hasError());
+        Assert.assertEquals(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, iter.getError().getType());
+        Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), iter.getError().getStatus().getCode());
+        Assert
+            .assertEquals(
+                "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+                iter.getError().getStatus().getErrorMsg());
+        assertEquals(6, iter.getIndex());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java
new file mode 100644
index 0000000..a722193
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/IteratorTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.LogManager;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class IteratorTest {
+
+    private IteratorImpl  iterImpl;
+    private Iterator      iter;
+
+    @Mock
+    private StateMachine  fsm;
+    @Mock
+    private LogManager    logManager;
+    private List closures;
+    private AtomicLong    applyingIndex;
+
+    @Before
+    public void setup() {
+        this.applyingIndex = new AtomicLong(0);
+        this.closures = new ArrayList<>();
+        for (int i = 0; i < 11; i++) {
+            this.closures.add(new MockClosure());
+            final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+            log.getId().setIndex(i);
+            log.getId().setTerm(1);
+            log.setData(ByteBuffer.allocate(i));
+            Mockito.when(this.logManager.getEntry(i)).thenReturn(log);
+        }
+        this.iterImpl = new IteratorImpl(fsm, logManager, closures, 0L, 0L, 10L, applyingIndex);
+        this.iter = new IteratorWrapper(iterImpl);
+    }
+
+    @Test
+    public void testPredicates() {
+        assertTrue(this.iter.hasNext());
+    }
+
+    @Test
+    public void testNext() {
+        int i = 1;
+        while (iter.hasNext()) {
+            assertEquals(i, iter.getIndex());
+            assertNotNull(iter.done());
+            assertEquals(i, iter.getIndex());
+            assertEquals(1, iter.getTerm());
+            assertEquals(i, iter.getData().remaining());
+            iter.next();
+            i++;
+        }
+        assertEquals(i, 11);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testSetErrorAndRollbackInvalid() {
+        this.iter.setErrorAndRollback(-1, null);
+    }
+
+    @Test
+    public void testSetErrorAndRollback() {
+        testNext();
+        assertFalse(iterImpl.hasError());
+        this.iter.setErrorAndRollback(5, new Status(-1, "test"));
+        assertTrue(iterImpl.hasError());
+        Assert.assertEquals(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, iterImpl.getError().getType());
+        Assert.assertEquals(RaftError.ESTATEMACHINE.getNumber(), iterImpl.getError().getStatus().getCode());
+        Assert
+            .assertEquals(
+                "StateMachine meet critical error when applying one or more tasks since index=6, Status[UNKNOWN<-1>: test]",
+                iterImpl.getError().getStatus().getErrorMsg());
+        assertEquals(6, iter.getIndex());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
new file mode 100644
index 0000000..2b67748
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+
+class MockClosure implements Closure {
+    Status s;
+
+    @Override
+    public void run(Status status) {
+        this.s = status;
+
+    }
+}
\ No newline at end of file
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
new file mode 100644
index 0000000..07d2c88
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+public class MockStateMachine extends StateMachineAdapter {
+
+    private final Lock             lock                  = new ReentrantLock();
+    private volatile int           onStartFollowingTimes = 0;
+    private volatile int           onStopFollowingTimes  = 0;
+    private volatile long          leaderTerm            = -1;
+    private volatile long          appliedIndex          = -1;
+    private volatile long          snapshotIndex         = -1L;
+    private final List logs                  = new ArrayList<>();
+    private final Endpoint         address;
+    private volatile int           saveSnapshotTimes;
+    private volatile int           loadSnapshotTimes;
+
+    public Endpoint getAddress() {
+        return this.address;
+    }
+
+    public MockStateMachine(final Endpoint address) {
+        super();
+        this.address = address;
+    }
+
+    public int getSaveSnapshotTimes() {
+        return this.saveSnapshotTimes;
+    }
+
+    public int getLoadSnapshotTimes() {
+        return this.loadSnapshotTimes;
+    }
+
+    public int getOnStartFollowingTimes() {
+        return this.onStartFollowingTimes;
+    }
+
+    public int getOnStopFollowingTimes() {
+        return this.onStopFollowingTimes;
+    }
+
+    public long getLeaderTerm() {
+        return this.leaderTerm;
+    }
+
+    public long getAppliedIndex() {
+        return this.appliedIndex;
+    }
+
+    public long getSnapshotIndex() {
+        return this.snapshotIndex;
+    }
+
+    public void lock() {
+        this.lock.lock();
+    }
+
+    public void unlock() {
+        this.lock.unlock();
+    }
+
+    public List getLogs() {
+        this.lock.lock();
+        try {
+            return this.logs;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    private final AtomicLong lastAppliedIndex = new AtomicLong(-1);
+
+    @Override
+    public void onApply(final Iterator iter) {
+        while (iter.hasNext()) {
+            this.lock.lock();
+            try {
+                if (iter.getIndex() <= this.lastAppliedIndex.get()) {
+                    //prevent duplication
+                    continue;
+                }
+                this.lastAppliedIndex.set(iter.getIndex());
+                this.logs.add(iter.getData().slice());
+                if (iter.done() != null) {
+                    iter.done().run(Status.OK());
+                }
+            } finally {
+                this.lock.unlock();
+            }
+            this.appliedIndex = iter.getIndex();
+            iter.next();
+        }
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm > 0;
+    }
+
+    @Override
+    public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+        this.saveSnapshotTimes++;
+        final String path = writer.getPath() + File.separator + "data";
+        final File file = new File(path);
+        try (FileOutputStream fout = new FileOutputStream(file);
+                BufferedOutputStream out = new BufferedOutputStream(fout)) {
+            this.lock.lock();
+            try {
+                for (final ByteBuffer buf : this.logs) {
+                    final byte[] bs = new byte[4];
+                    Bits.putInt(bs, 0, buf.remaining());
+                    out.write(bs);
+                    out.write(buf.array());
+                }
+                this.snapshotIndex = this.appliedIndex;
+            } finally {
+                this.lock.unlock();
+            }
+            System.out.println("Node<" + this.address + "> saved snapshot into " + file);
+            writer.addFile("data");
+            done.run(Status.OK());
+        } catch (final IOException e) {
+            e.printStackTrace();
+            done.run(new Status(RaftError.EIO, "Fail to save snapshot"));
+        }
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        SnapshotMeta meta = reader.load();
+        this.lastAppliedIndex.set(meta.getLastIncludedIndex());
+        this.loadSnapshotTimes++;
+        final String path = reader.getPath() + File.separator + "data";
+        final File file = new File(path);
+        if (!file.exists()) {
+            return false;
+        }
+        try (FileInputStream fin = new FileInputStream(file); BufferedInputStream in = new BufferedInputStream(fin)) {
+            this.lock.lock();
+            this.logs.clear();
+            try {
+                while (true) {
+                    final byte[] bs = new byte[4];
+                    if (in.read(bs) == 4) {
+                        final int len = Bits.getInt(bs, 0);
+                        final byte[] buf = new byte[len];
+                        if (in.read(buf) != len) {
+                            break;
+                        }
+                        this.logs.add(ByteBuffer.wrap(buf));
+                    } else {
+                        break;
+                    }
+                }
+            } finally {
+                this.lock.unlock();
+            }
+            System.out.println("Node<" + this.address + "> loaded snapshot from " + path);
+            return true;
+        } catch (final IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        super.onLeaderStart(term);
+        this.leaderTerm = term;
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        super.onLeaderStop(status);
+        this.leaderTerm = -1;
+    }
+
+    @Override
+    public void onStopFollowing(final LeaderChangeContext ctx) {
+        super.onStopFollowing(ctx);
+        this.onStopFollowingTimes++;
+    }
+
+    @Override
+    public void onStartFollowing(final LeaderChangeContext ctx) {
+        super.onStartFollowing(ctx);
+        this.onStartFollowingTimes++;
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java
new file mode 100644
index 0000000..500359f
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/NodeTest.java
@@ -0,0 +1,3429 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Vector;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+import org.rocksdb.util.SizeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.StateMachine;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.JoinableClosure;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.closure.SynchronizedClosure;
+import com.alipay.sofa.jraft.closure.TaskClosure;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.entity.UserLog;
+import com.alipay.sofa.jraft.error.LogIndexOutOfBoundsException;
+import com.alipay.sofa.jraft.error.LogNotFoundException;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.BootstrapOptions;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.ThroughputSnapshotThrottle;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.StorageOptionsFactory;
+import com.alipay.sofa.jraft.util.Utils;
+import com.codahale.metrics.ConsoleReporter;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class NodeTest {
+
+    static final Logger         LOG            = LoggerFactory.getLogger(NodeTest.class);
+
+    private String              dataPath;
+
+    private final AtomicInteger startedCounter = new AtomicInteger(0);
+    private final AtomicInteger stoppedCounter = new AtomicInteger(0);
+
+    @Rule
+    public TestName             testName       = new TestName();
+
+    private long                testStartMs;
+
+    private static DumpThread   dumpThread;
+
+    static class DumpThread extends Thread {
+        private static long      DUMP_TIMEOUT_MS = 5 * 60 * 1000;
+        private volatile boolean stopped         = false;
+
+        @Override
+        public void run() {
+            while (!this.stopped) {
+                try {
+                    Thread.sleep(DUMP_TIMEOUT_MS);
+                    System.out.println("Test hang too long, dump threads");
+                    TestUtils.dumpThreads();
+                } catch (InterruptedException e) {
+                    // reset request, continue
+                    continue;
+                }
+            }
+        }
+    }
+
+    @BeforeClass
+    public static void setupNodeTest() {
+        StorageOptionsFactory.registerRocksDBTableFormatConfig(RocksDBLogStorage.class, StorageOptionsFactory
+            .getDefaultRocksDBTableConfig().setBlockCacheSize(256 * SizeUnit.MB));
+        dumpThread = new DumpThread();
+        dumpThread.setName("NodeTest-DumpThread");
+        dumpThread.setDaemon(true);
+        dumpThread.start();
+    }
+
+    @AfterClass
+    public static void tearNodeTest() throws Exception {
+        dumpThread.stopped = true;
+        dumpThread.interrupt();
+        dumpThread.join(100);
+    }
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+        this.dataPath = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.dataPath));
+        assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        this.testStartMs = Utils.monotonicMs();
+        dumpThread.interrupt(); // reset dump timeout
+    }
+
+    @After
+    public void teardown() throws Exception {
+        if (!TestCluster.CLUSTERS.isEmpty()) {
+            for (final TestCluster c : TestCluster.CLUSTERS.removeAll()) {
+                c.stopAll();
+            }
+        }
+        if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+            Thread.sleep(5000);
+            assertEquals(0, NodeImpl.GLOBAL_NUM_NODES.get());
+        }
+        FileUtils.deleteDirectory(new File(this.dataPath));
+        NodeManager.getInstance().clear();
+        this.startedCounter.set(0);
+        this.stoppedCounter.set(0);
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName() + ", cost:"
+                           + (Utils.monotonicMs() - this.testStartMs) + " ms.");
+    }
+
+    @Test
+    public void testInitShutdown() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setFsm(new MockStateMachine(addr));
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+
+        final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+        assertTrue(node.init(nodeOptions));
+
+        node.shutdown();
+        node.join();
+    }
+
+    @Test
+    public void testNodeTaskOverload() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final PeerId peer = new PeerId(addr, 0);
+
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+        final RaftOptions raftOptions = new RaftOptions();
+        raftOptions.setDisruptorBufferSize(2);
+        nodeOptions.setRaftOptions(raftOptions);
+        final MockStateMachine fsm = new MockStateMachine(addr);
+        nodeOptions.setFsm(fsm);
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+        final Node node = new NodeImpl("unittest", peer);
+        assertTrue(node.init(nodeOptions));
+
+        assertEquals(1, node.listPeers().size());
+        assertTrue(node.listPeers().contains(peer));
+
+        while (!node.isLeader()) {
+            ;
+        }
+
+        final List tasks = new ArrayList<>();
+        final AtomicInteger c = new AtomicInteger(0);
+        for (int i = 0; i < 10; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new JoinableClosure(status -> {
+                System.out.println(status);
+                if (!status.isOk()) {
+                    assertTrue(
+                            status.getRaftError() == RaftError.EBUSY || status.getRaftError() == RaftError.EPERM);
+                }
+                c.incrementAndGet();
+            }));
+            node.apply(task);
+            tasks.add(task);
+        }
+        try {
+            Task.joinAll(tasks, TimeUnit.SECONDS.toMillis(30));
+            assertEquals(10, c.get());
+        } finally {
+            node.shutdown();
+            node.join();
+        }
+    }
+
+    /**
+     * Test rollback stateMachine with readIndex for issue 317:
+     * https://github.com/sofastack/sofa-jraft/issues/317
+     */
+    @Test
+    public void testRollbackStateMachineWithReadIndex_Issue317() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final PeerId peer = new PeerId(addr, 0);
+
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+        final CountDownLatch applyCompleteLatch = new CountDownLatch(1);
+        final CountDownLatch applyLatch = new CountDownLatch(1);
+        final CountDownLatch readIndexLatch = new CountDownLatch(1);
+        final AtomicInteger currentValue = new AtomicInteger(-1);
+        final String errorMsg = this.testName.getMethodName();
+        final StateMachine fsm = new StateMachineAdapter() {
+
+            @Override
+            public void onApply(final Iterator iter) {
+                // Notify that the #onApply is preparing to go.
+                readIndexLatch.countDown();
+                // Wait for submitting a read-index request
+                try {
+                    applyLatch.await();
+                } catch (InterruptedException e) {
+                    fail();
+                }
+                int i = 0;
+                while (iter.hasNext()) {
+                    byte[] data = iter.next().array();
+                    int v = Bits.getInt(data, 0);
+                    assertEquals(i++, v);
+                    currentValue.set(v);
+                }
+                if (i > 0) {
+                    // rollback
+                    currentValue.set(i - 1);
+                    iter.setErrorAndRollback(1, new Status(-1, errorMsg));
+                    applyCompleteLatch.countDown();
+                }
+            }
+        };
+        nodeOptions.setFsm(fsm);
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+        final Node node = new NodeImpl("unittest", peer);
+        assertTrue(node.init(nodeOptions));
+
+        assertEquals(1, node.listPeers().size());
+        assertTrue(node.listPeers().contains(peer));
+
+        while (!node.isLeader()) {
+            ;
+        }
+
+        int n = 5;
+        {
+            // apply tasks
+            for (int i = 0; i < n; i++) {
+                byte[] b = new byte[4];
+                Bits.putInt(b, 0, i);
+                node.apply(new Task(ByteBuffer.wrap(b), null));
+            }
+        }
+
+        final AtomicInteger readIndexSuccesses = new AtomicInteger(0);
+        {
+            // Submit a read-index, wait for #onApply
+            readIndexLatch.await();
+            final CountDownLatch latch = new CountDownLatch(1);
+            node.readIndex(null, new ReadIndexClosure() {
+
+                @Override
+                public void run(final Status status, final long index, final byte[] reqCtx) {
+                    try {
+                        if (status.isOk()) {
+                            readIndexSuccesses.incrementAndGet();
+                        } else {
+                            assertTrue("Unexpected status: " + status,
+                                status.getErrorMsg().contains(errorMsg) || status.getRaftError() == RaftError.ETIMEDOUT
+                                        || status.getErrorMsg().contains("Invalid state for readIndex: STATE_ERROR"));
+                        }
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+            });
+            // We have already submit a read-index request,
+            // notify #onApply can go right now
+            applyLatch.countDown();
+
+            // The state machine is in error state, the node should step down.
+            while (node.isLeader()) {
+                Thread.sleep(10);
+            }
+            latch.await();
+            applyCompleteLatch.await();
+        }
+        // No read-index request succeed.
+        assertEquals(0, readIndexSuccesses.get());
+        assertTrue(n - 1 >= currentValue.get());
+
+        node.shutdown();
+        node.join();
+    }
+
+    @Test
+    public void testSingleNode() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final PeerId peer = new PeerId(addr, 0);
+
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+        final MockStateMachine fsm = new MockStateMachine(addr);
+        nodeOptions.setFsm(fsm);
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer)));
+        final Node node = new NodeImpl("unittest", peer);
+        assertTrue(node.init(nodeOptions));
+
+        assertEquals(1, node.listPeers().size());
+        assertTrue(node.listPeers().contains(peer));
+
+        while (!node.isLeader()) {
+            ;
+        }
+
+        sendTestTaskAndWait(node);
+        assertEquals(10, fsm.getLogs().size());
+        int i = 0;
+        for (final ByteBuffer data : fsm.getLogs()) {
+            assertEquals("hello" + i++, new String(data.array()));
+        }
+        node.shutdown();
+        node.join();
+    }
+
+    @Test
+    public void testNoLeader() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+
+        assertTrue(cluster.start(peers.get(0).getEndpoint()));
+
+        final List followers = cluster.getFollowers();
+        assertEquals(1, followers.size());
+
+        final Node follower = followers.get(0);
+        sendTestTaskAndWait(follower, 0, RaftError.EPERM);
+
+        // adds a peer3
+        final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+        CountDownLatch latch = new CountDownLatch(1);
+        follower.addPeer(peer3, new ExpectClosure(RaftError.EPERM, latch));
+        waitLatch(latch);
+
+        // remove the peer0
+        final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        latch = new CountDownLatch(1);
+        follower.removePeer(peer0, new ExpectClosure(RaftError.EPERM, latch));
+        waitLatch(latch);
+
+        cluster.stopAll();
+    }
+
+    private void sendTestTaskAndWait(final Node node) throws InterruptedException {
+        this.sendTestTaskAndWait(node, 0, RaftError.SUCCESS);
+    }
+
+    private void sendTestTaskAndWait(final Node node, final int start, final RaftError err) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(10);
+        for (int i = start; i < start + 10; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(err, latch));
+            node.apply(task);
+        }
+        waitLatch(latch);
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private int sendTestTaskAndWait(final String prefix, final Node node, final int code) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(10);
+        final AtomicInteger successCount = new AtomicInteger(0);
+        for (int i = 0; i < 10; i++) {
+            final ByteBuffer data = ByteBuffer.wrap((prefix + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(code, null, latch, successCount));
+            node.apply(task);
+        }
+        waitLatch(latch);
+        return successCount.get();
+    }
+
+    @Test
+    public void testTripleNodesWithReplicatorStateListener() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        final UserReplicatorStateListener listener1 = new UserReplicatorStateListener();
+        final UserReplicatorStateListener listener2 = new UserReplicatorStateListener();
+
+        for (Node node : cluster.getNodes()) {
+            node.addReplicatorStateListener(listener1);
+            node.addReplicatorStateListener(listener2);
+
+        }
+        // elect leader
+        cluster.waitLeader();
+        assertEquals(4, this.startedCounter.get());
+        assertEquals(2, cluster.getLeader().getReplicatorStatueListeners().size());
+        assertEquals(2, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+        assertEquals(2, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+        for (Node node : cluster.getNodes()) {
+            node.removeReplicatorStateListener(listener1);
+        }
+        assertEquals(1, cluster.getLeader().getReplicatorStatueListeners().size());
+        assertEquals(1, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+        assertEquals(1, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testVoteTimedoutStepDown() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        // Stop all followers
+        List followers = cluster.getFollowers();
+        assertFalse(followers.isEmpty());
+        for (Node node : followers) {
+            assertTrue(cluster.stop(node.getNodeId().getPeerId().getEndpoint()));
+        }
+
+        // Wait leader to step down.
+        while (leader.isLeader()) {
+            Thread.sleep(10);
+        }
+
+        // old leader try to elect self, it should fail.
+        ((NodeImpl) leader).tryElectSelf();
+        Thread.sleep(1500);
+        // Start followers
+        for (Node node : followers) {
+            assertTrue(cluster.start(node.getNodeId().getPeerId().getEndpoint()));
+        }
+
+        cluster.ensureSame(-1);
+        cluster.stopAll();
+    }
+
+    class UserReplicatorStateListener implements Replicator.ReplicatorStateListener {
+        @Override
+        public void onCreated(final PeerId peer) {
+            LOG.info("Replicator has created");
+            NodeTest.this.startedCounter.incrementAndGet();
+        }
+
+        @Override
+        public void stateChanged(final PeerId peer, final ReplicatorState newState) {
+            LOG.info("Replicator {} state is changed into {}.", peer, newState);
+        }
+
+        @Override
+        public void onError(final PeerId peer, final Status status) {
+            LOG.info("Replicator has errors");
+        }
+
+        @Override
+        public void onDestroyed(final PeerId peer) {
+            LOG.info("Replicator has been destroyed");
+            NodeTest.this.stoppedCounter.incrementAndGet();
+        }
+    }
+
+    @Test
+    public void testLeaderTransferWithReplicatorStateListener() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+        cluster.waitLeader();
+        final UserReplicatorStateListener listener = new UserReplicatorStateListener();
+        for (Node node : cluster.getNodes()) {
+            node.addReplicatorStateListener(listener);
+        }
+        Node leader = cluster.getLeader();
+        this.sendTestTaskAndWait(leader);
+        Thread.sleep(100);
+        final List followers = cluster.getFollowers();
+
+        final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+        LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+        assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+        Thread.sleep(1000);
+        cluster.waitLeader();
+        assertEquals(2, this.startedCounter.get());
+
+        for (Node node : cluster.getNodes()) {
+            node.clearReplicatorStateListeners();
+        }
+        assertEquals(0, cluster.getLeader().getReplicatorStatueListeners().size());
+        assertEquals(0, cluster.getFollowers().get(0).getReplicatorStatueListeners().size());
+        assertEquals(0, cluster.getFollowers().get(1).getReplicatorStatueListeners().size());
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testTripleNodes() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        {
+            final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+            final Task task = new Task(data, null);
+            leader.apply(task);
+        }
+
+        {
+            // task with TaskClosure
+            final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+            final Vector cbs = new Vector<>();
+            final CountDownLatch latch = new CountDownLatch(1);
+            final Task task = new Task(data, new TaskClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    cbs.add("apply");
+                    latch.countDown();
+                }
+
+                @Override
+                public void onCommitted() {
+                    cbs.add("commit");
+
+                }
+            });
+            leader.apply(task);
+            latch.await();
+            assertEquals(2, cbs.size());
+            assertEquals("commit", cbs.get(0));
+            assertEquals("apply", cbs.get(1));
+        }
+
+        cluster.ensureSame(-1);
+        assertEquals(2, cluster.getFollowers().size());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testSingleNodeWithLearner() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final PeerId peer = new PeerId(addr, 0);
+
+        final Endpoint learnerAddr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT + 1);
+        final PeerId learnerPeer = new PeerId(learnerAddr, 0);
+
+        NodeManager.getInstance().addAddress(addr);
+        NodeManager.getInstance().addAddress(learnerAddr);
+        MockStateMachine learnerFsm = null;
+        Node learner = null;
+        RaftGroupService learnerServer = null;
+        {
+            // Start learner
+            final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+            learnerFsm = new MockStateMachine(learnerAddr);
+            nodeOptions.setFsm(learnerFsm);
+            nodeOptions.setLogUri(this.dataPath + File.separator + "log1");
+            nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta1");
+            nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot1");
+            nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), Collections
+                .singletonList(learnerPeer)));
+
+            final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(learnerAddr);
+            learnerServer = new RaftGroupService("unittest", new PeerId(learnerAddr, 0), nodeOptions, rpcServer);
+            learner = learnerServer.start();
+        }
+
+        {
+            // Start leader
+            final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+            final MockStateMachine fsm = new MockStateMachine(addr);
+            nodeOptions.setFsm(fsm);
+            nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+            nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+            nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+            nodeOptions.setInitialConf(new Configuration(Collections.singletonList(peer), Collections
+                .singletonList(learnerPeer)));
+            final Node node = new NodeImpl("unittest", peer);
+            assertTrue(node.init(nodeOptions));
+
+            assertEquals(1, node.listPeers().size());
+            assertTrue(node.listPeers().contains(peer));
+            while (!node.isLeader()) {
+                ;
+            }
+            sendTestTaskAndWait(node);
+            assertEquals(10, fsm.getLogs().size());
+            int i = 0;
+            for (final ByteBuffer data : fsm.getLogs()) {
+                assertEquals("hello" + i++, new String(data.array()));
+            }
+            Thread.sleep(1000); //wait for entries to be replicated to learner.
+            node.shutdown();
+            node.join();
+        }
+        {
+            // assert learner fsm
+            assertEquals(10, learnerFsm.getLogs().size());
+            int i = 0;
+            for (final ByteBuffer data : learnerFsm.getLogs()) {
+                assertEquals("hello" + i++, new String(data.array()));
+            }
+            learnerServer.shutdown();
+            learnerServer.join();
+        }
+    }
+
+    @Test
+    public void testResetLearners() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final LinkedHashSet learners = new LinkedHashSet<>();
+
+        for (int i = 0; i < 3; i++) {
+            learners.add(new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3 + i));
+        }
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers, learners, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+        for (final PeerId peer : learners) {
+            assertTrue(cluster.startLearner(peer));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+
+        assertEquals(3, leader.listAlivePeers().size());
+        assertEquals(3, leader.listAliveLearners().size());
+
+        this.sendTestTaskAndWait(leader);
+        Thread.sleep(500);
+        List fsms = cluster.getFsms();
+        assertEquals(6, fsms.size());
+        cluster.ensureSame();
+
+        {
+            // Reset learners to 2 nodes
+            PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+            learners.remove(learnerPeer);
+            assertEquals(2, learners.size());
+
+            SynchronizedClosure done = new SynchronizedClosure();
+            leader.resetLearners(new ArrayList<>(learners), done);
+            assertTrue(done.await().isOk());
+            assertEquals(2, leader.listAliveLearners().size());
+            assertEquals(2, leader.listLearners().size());
+            this.sendTestTaskAndWait(leader);
+            Thread.sleep(500);
+
+            assertEquals(6, fsms.size());
+
+            MockStateMachine fsm = fsms.remove(3); // get the removed learner's fsm
+            assertEquals(fsm.getAddress(), learnerPeer.getEndpoint());
+            // Ensure no more logs replicated to the removed learner.
+            assertTrue(cluster.getLeaderFsm().getLogs().size() > fsm.getLogs().size());
+            assertEquals(cluster.getLeaderFsm().getLogs().size(), 2 * fsm.getLogs().size());
+        }
+        {
+            // remove another learner
+            PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 4);
+            SynchronizedClosure done = new SynchronizedClosure();
+            leader.removeLearners(Arrays.asList(learnerPeer), done);
+            assertTrue(done.await().isOk());
+
+            this.sendTestTaskAndWait(leader);
+            Thread.sleep(500);
+            MockStateMachine fsm = fsms.remove(3); // get the removed learner's fsm
+            assertEquals(fsm.getAddress(), learnerPeer.getEndpoint());
+            // Ensure no more logs replicated to the removed learner.
+            assertTrue(cluster.getLeaderFsm().getLogs().size() > fsm.getLogs().size());
+            assertEquals(cluster.getLeaderFsm().getLogs().size(), fsm.getLogs().size() / 2 * 3);
+        }
+
+        assertEquals(3, leader.listAlivePeers().size());
+        assertEquals(1, leader.listAliveLearners().size());
+        assertEquals(1, leader.listLearners().size());
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testTripleNodesWithStaticLearners() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        LinkedHashSet learners = new LinkedHashSet<>();
+        PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+        learners.add(learnerPeer);
+        cluster.setLearners(learners);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+        final Node leader = cluster.getLeader();
+
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(leader.listLearners().size(), 1);
+        assertTrue(leader.listLearners().contains(learnerPeer));
+        assertTrue(leader.listAliveLearners().isEmpty());
+
+        // start learner after cluster setup.
+        assertTrue(cluster.start(learnerPeer.getEndpoint()));
+
+        Thread.sleep(1000);
+
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(leader.listLearners().size(), 1);
+        assertEquals(leader.listAliveLearners().size(), 1);
+
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+        assertEquals(4, cluster.getFsms().size());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testTripleNodesWithLearners() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        assertTrue(leader.listLearners().isEmpty());
+        assertTrue(leader.listAliveLearners().isEmpty());
+
+        {
+            // Adds a learner
+            SynchronizedClosure done = new SynchronizedClosure();
+            PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+            // Start learner
+            assertTrue(cluster.startLearner(learnerPeer));
+            leader.addLearners(Arrays.asList(learnerPeer), done);
+            assertTrue(done.await().isOk());
+            assertEquals(1, leader.listAliveLearners().size());
+            assertEquals(1, leader.listLearners().size());
+        }
+
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        {
+            final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+            final Task task = new Task(data, null);
+            leader.apply(task);
+        }
+
+        {
+            // task with TaskClosure
+            final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+            final Vector cbs = new Vector<>();
+            final CountDownLatch latch = new CountDownLatch(1);
+            final Task task = new Task(data, new TaskClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    cbs.add("apply");
+                    latch.countDown();
+                }
+
+                @Override
+                public void onCommitted() {
+                    cbs.add("commit");
+
+                }
+            });
+            leader.apply(task);
+            latch.await();
+            assertEquals(2, cbs.size());
+            assertEquals("commit", cbs.get(0));
+            assertEquals("apply", cbs.get(1));
+        }
+
+        assertEquals(4, cluster.getFsms().size());
+        assertEquals(2, cluster.getFollowers().size());
+        assertEquals(1, cluster.getLearners().size());
+        cluster.ensureSame(-1);
+
+        {
+            // Adds another learner
+            SynchronizedClosure done = new SynchronizedClosure();
+            PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 4);
+            // Start learner
+            assertTrue(cluster.startLearner(learnerPeer));
+            leader.addLearners(Arrays.asList(learnerPeer), done);
+            assertTrue(done.await().isOk());
+            assertEquals(2, leader.listAliveLearners().size());
+            assertEquals(2, leader.listLearners().size());
+        }
+        {
+            // stop two followers
+            for (Node follower : cluster.getFollowers()) {
+                assertTrue(cluster.stop(follower.getNodeId().getPeerId().getEndpoint()));
+            }
+            // send a new task
+            final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+            SynchronizedClosure done = new SynchronizedClosure();
+            leader.apply(new Task(data, done));
+            // should fail
+            assertFalse(done.await().isOk());
+            assertEquals(RaftError.EPERM, done.getStatus().getRaftError());
+            // One peer with two learners.
+            assertEquals(3, cluster.getFsms().size());
+            cluster.ensureSame(-1);
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNodesWithPriorityElection() throws Exception {
+
+        List priorities = new ArrayList<>();
+        priorities.add(100);
+        priorities.add(40);
+        priorities.add(40);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(100, leader.getNodeTargetPriority());
+        assertEquals(100, leader.getLeaderId().getPriority());
+        assertEquals(2, cluster.getFollowers().size());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNodesWithPartPriorityElection() throws Exception {
+
+        List priorities = new ArrayList<>();
+        priorities.add(100);
+        priorities.add(40);
+        priorities.add(-1);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(2, cluster.getFollowers().size());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNodesWithSpecialPriorityElection() throws Exception {
+
+        List priorities = new ArrayList();
+        priorities.add(0);
+        priorities.add(0);
+        priorities.add(-1);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(2, cluster.getFollowers().size());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNodesWithZeroValPriorityElection() throws Exception {
+
+        List priorities = new ArrayList();
+        priorities.add(50);
+        priorities.add(0);
+        priorities.add(0);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        assertEquals(2, cluster.getFollowers().size());
+        assertEquals(50, leader.getNodeTargetPriority());
+        assertEquals(50, leader.getLeaderId().getPriority());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNoLeaderWithZeroValPriorityElection() throws Exception {
+        List priorities = new ArrayList<>();
+        priorities.add(0);
+        priorities.add(0);
+        priorities.add(0);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        Thread.sleep(200);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(3, followers.size());
+
+        for (Node follower : followers) {
+            assertEquals(0, follower.getNodeId().getPeerId().getPriority());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testLeaderStopAndReElectWithPriority() throws Exception {
+        final List priorities = new ArrayList<>();
+        priorities.add(100);
+        priorities.add(60);
+        priorities.add(10);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        cluster.waitLeader();
+        Node leader = cluster.getLeader();
+
+        assertNotNull(leader);
+        assertEquals(100, leader.getNodeId().getPeerId().getPriority());
+        assertEquals(100, leader.getNodeTargetPriority());
+
+        // apply tasks to leader
+        sendTestTaskAndWait(leader);
+
+        // stop leader
+        assertTrue(cluster.stop(leader.getNodeId().getPeerId().getEndpoint()));
+
+        // elect new leader
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+
+        assertNotNull(leader);
+
+        // get current leader priority value
+        int leaderPriority = leader.getNodeId().getPeerId().getPriority();
+
+        // get current leader log size
+        int peer1LogSize = cluster.getFsmByPeer(peers.get(1)).getLogs().size();
+        int peer2LogSize = cluster.getFsmByPeer(peers.get(2)).getLogs().size();
+
+        // if the leader is lower priority value
+        if (leaderPriority == 10) {
+            // we just compare the two peers' log size value;
+            assertTrue(peer2LogSize > peer1LogSize);
+        } else {
+            assertEquals(60, leader.getNodeId().getPeerId().getPriority());
+            assertEquals(100, leader.getNodeTargetPriority());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testRemoveLeaderWithPriority() throws Exception {
+        final List priorities = new ArrayList();
+        priorities.add(100);
+        priorities.add(60);
+        priorities.add(10);
+
+        final List peers = TestUtils.generatePriorityPeers(3, priorities);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), peer.getPriority()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(100, leader.getNodeTargetPriority());
+        assertEquals(100, leader.getNodeId().getPeerId().getPriority());
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId oldLeader = leader.getNodeId().getPeerId().copy();
+        final Endpoint oldLeaderAddr = oldLeader.getEndpoint();
+
+        // remove old leader
+        LOG.info("Remove old leader {}", oldLeader);
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.removePeer(oldLeader, new ExpectClosure(latch));
+        waitLatch(latch);
+        assertEquals(60, leader.getNodeTargetPriority());
+
+        // stop and clean old leader
+        LOG.info("Stop and clean old leader {}", oldLeader);
+        assertTrue(cluster.stop(oldLeaderAddr));
+        cluster.clean(oldLeaderAddr);
+
+        // elect new leader
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        LOG.info("New leader is {}", leader);
+        assertNotNull(leader);
+        assertNotSame(leader, oldLeader);
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testTripleNodesV1V2Codec() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (int i = 0; i < peers.size(); i++) {
+            // Peer3 use codec v1
+            if (i == 2) {
+                cluster.setRaftServiceFactory(new V1JRaftServiceFactory());
+            }
+            assertTrue(cluster.start(peers.get(i).getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        {
+            final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+            final Task task = new Task(data, null);
+            leader.apply(task);
+        }
+
+        {
+            // task with TaskClosure
+            final ByteBuffer data = ByteBuffer.wrap("task closure".getBytes());
+            final Vector cbs = new Vector<>();
+            final CountDownLatch latch = new CountDownLatch(1);
+            final Task task = new Task(data, new TaskClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    cbs.add("apply");
+                    latch.countDown();
+                }
+
+                @Override
+                public void onCommitted() {
+                    cbs.add("commit");
+
+                }
+            });
+            leader.apply(task);
+            latch.await();
+            assertEquals(2, cbs.size());
+            assertEquals("commit", cbs.get(0));
+            assertEquals("apply", cbs.get(1));
+        }
+
+        cluster.ensureSame(-1);
+        assertEquals(2, cluster.getFollowers().size());
+
+        // transfer the leader to v1 codec peer
+        assertTrue(leader.transferLeadershipTo(peers.get(2)).isOk());
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(leader.getLeaderId(), peers.get(2));
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+        cluster.ensureSame();
+        cluster.stopAll();
+
+        // start the cluster with v2 codec, should work
+        final TestCluster newCluster = new TestCluster("unittest", this.dataPath, peers);
+        for (int i = 0; i < peers.size(); i++) {
+            assertTrue(newCluster.start(peers.get(i).getEndpoint()));
+        }
+
+        // elect leader
+        newCluster.waitLeader();
+        newCluster.ensureSame();
+        leader = newCluster.getLeader();
+        assertNotNull(leader);
+        // apply new tasks
+        this.sendTestTaskAndWait(leader);
+        newCluster.ensureSame();
+        newCluster.stopAll();
+    }
+
+    @Test
+    public void testChecksum() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        // start with checksum validation
+        {
+            final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+            final RaftOptions raftOptions = new RaftOptions();
+            raftOptions.setEnableLogEntryChecksum(true);
+            for (final PeerId peer : peers) {
+                assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+            }
+
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            assertNotNull(leader);
+            assertEquals(3, leader.listPeers().size());
+            this.sendTestTaskAndWait(leader);
+            cluster.ensureSame();
+
+            cluster.stopAll();
+        }
+
+        // restart with peer3 enable checksum validation
+        {
+            final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+            RaftOptions raftOptions = new RaftOptions();
+            raftOptions.setEnableLogEntryChecksum(false);
+            for (final PeerId peer : peers) {
+                if (peer.equals(peers.get(2))) {
+                    raftOptions = new RaftOptions();
+                    raftOptions.setEnableLogEntryChecksum(true);
+                }
+                assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+            }
+
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            assertNotNull(leader);
+            assertEquals(3, leader.listPeers().size());
+            this.sendTestTaskAndWait(leader);
+            cluster.ensureSame();
+
+            cluster.stopAll();
+        }
+
+        // restart with no checksum validation
+        {
+            final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+            final RaftOptions raftOptions = new RaftOptions();
+            raftOptions.setEnableLogEntryChecksum(false);
+            for (final PeerId peer : peers) {
+                assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+            }
+
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            assertNotNull(leader);
+            assertEquals(3, leader.listPeers().size());
+            this.sendTestTaskAndWait(leader);
+            cluster.ensureSame();
+
+            cluster.stopAll();
+        }
+
+        // restart with all peers enable checksum validation
+        {
+            final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+            final RaftOptions raftOptions = new RaftOptions();
+            raftOptions.setEnableLogEntryChecksum(true);
+            for (final PeerId peer : peers) {
+                assertTrue(cluster.start(peer.getEndpoint(), false, 300, true, null, raftOptions));
+            }
+
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            assertNotNull(leader);
+            assertEquals(3, leader.listPeers().size());
+            this.sendTestTaskAndWait(leader);
+            cluster.ensureSame();
+
+            cluster.stopAll();
+        }
+
+    }
+
+    @Test
+    public void testReadIndex() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        // first call will fail-fast when no connection
+        if (!assertReadIndex(leader, 11)) {
+            assertTrue(assertReadIndex(leader, 11));
+        }
+
+        // read from follower
+        for (final Node follower : cluster.getFollowers()) {
+            assertNotNull(follower);
+            assertReadIndex(follower, 11);
+        }
+
+        // read with null request context
+        final CountDownLatch latch = new CountDownLatch(1);
+        leader.readIndex(null, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertNull(reqCtx);
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+        latch.await();
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testReadIndexTimeout() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        sendTestTaskAndWait(leader);
+
+        // first call will fail-fast when no connection
+        if (!assertReadIndex(leader, 11)) {
+            assertTrue(assertReadIndex(leader, 11));
+        }
+
+        // read from follower
+        for (final Node follower : cluster.getFollowers()) {
+            assertNotNull(follower);
+            assertReadIndex(follower, 11);
+        }
+
+        // read with null request context
+        final CountDownLatch latch = new CountDownLatch(1);
+        final long start = System.currentTimeMillis();
+        leader.readIndex(null, new ReadIndexClosure(0) {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertNull(reqCtx);
+                if (status.isOk()) {
+                    System.err.println("Read-index so fast: " + (System.currentTimeMillis() - start) + "ms");
+                } else {
+                    assertEquals(status, new Status(RaftError.ETIMEDOUT, "read-index request timeout"));
+                    assertEquals(index, -1);
+                }
+                latch.countDown();
+            }
+        });
+        latch.await();
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testReadIndexFromLearner() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        {
+            // Adds a learner
+            SynchronizedClosure done = new SynchronizedClosure();
+            PeerId learnerPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+            // Start learner
+            assertTrue(cluster.startLearner(learnerPeer));
+            leader.addLearners(Arrays.asList(learnerPeer), done);
+            assertTrue(done.await().isOk());
+            assertEquals(1, leader.listAliveLearners().size());
+            assertEquals(1, leader.listLearners().size());
+        }
+
+        Thread.sleep(100);
+        // read from learner
+        Node learner = cluster.getNodes().get(3);
+        assertNotNull(leader);
+        assertReadIndex(learner, 12);
+        assertReadIndex(learner, 12);
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testReadIndexChaos() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+
+        final CountDownLatch latch = new CountDownLatch(10);
+        for (int i = 0; i < 10; i++) {
+            new Thread() {
+                @Override
+                public void run() {
+                    try {
+                        for (int i = 0; i < 100; i++) {
+                            try {
+                                sendTestTaskAndWait(leader);
+                            } catch (final InterruptedException e) {
+                                Thread.currentThread().interrupt();
+                            }
+                            readIndexRandom(cluster);
+                        }
+                    } finally {
+                        latch.countDown();
+                    }
+                }
+
+                private void readIndexRandom(final TestCluster cluster) {
+                    final CountDownLatch readLatch = new CountDownLatch(1);
+                    final byte[] requestContext = TestUtils.getRandomBytes();
+                    cluster.getNodes().get(ThreadLocalRandom.current().nextInt(3))
+                        .readIndex(requestContext, new ReadIndexClosure() {
+
+                            @Override
+                            public void run(final Status status, final long index, final byte[] reqCtx) {
+                                if (status.isOk()) {
+                                    assertTrue(status.toString(), status.isOk());
+                                    assertTrue(index > 0);
+                                    assertArrayEquals(requestContext, reqCtx);
+                                }
+                                readLatch.countDown();
+                            }
+                        });
+                    try {
+                        readLatch.await();
+                    } catch (final InterruptedException e) {
+                        Thread.currentThread().interrupt();
+                    }
+                }
+            }.start();
+        }
+
+        latch.await();
+
+        cluster.ensureSame();
+
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(10000, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @SuppressWarnings({ "unused", "SameParameterValue" })
+    private boolean assertReadIndex(final Node node, final int index) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        final AtomicBoolean success = new AtomicBoolean(false);
+        node.readIndex(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long theIndex, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    assertEquals(index, theIndex);
+                    assertArrayEquals(requestContext, reqCtx);
+                    success.set(true);
+                } else {
+                    assertTrue(status.getErrorMsg(), status.getErrorMsg().contains("RPC exception:Check connection["));
+                    assertTrue(status.getErrorMsg(), status.getErrorMsg().contains("] fail and try to create new one"));
+                }
+                latch.countDown();
+            }
+        });
+        latch.await();
+        return success.get();
+    }
+
+    @Test
+    public void testNodeMetrics() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 300, true));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertEquals(3, leader.listPeers().size());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        {
+            final ByteBuffer data = ByteBuffer.wrap("no closure".getBytes());
+            final Task task = new Task(data, null);
+            leader.apply(task);
+        }
+
+        cluster.ensureSame(-1);
+        for (final Node node : cluster.getNodes()) {
+            System.out.println("-------------" + node.getNodeId() + "-------------");
+            final ConsoleReporter reporter = ConsoleReporter.forRegistry(node.getNodeMetrics().getMetricRegistry())
+                .build();
+            reporter.report();
+            reporter.close();
+            System.out.println();
+        }
+        // TODO check http status
+        assertEquals(2, cluster.getFollowers().size());
+        cluster.stopAll();
+        //   System.out.println(node.getNodeMetrics().getMetrics());
+    }
+
+    @Test
+    public void testLeaderFail() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        LOG.info("Current leader is {}", leader.getLeaderId());
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        // stop leader
+        LOG.warn("Stop leader {}", leader.getNodeId().getPeerId());
+        final PeerId oldLeader = leader.getNodeId().getPeerId();
+        assertTrue(cluster.stop(leader.getNodeId().getPeerId().getEndpoint()));
+
+        // apply something when follower
+        final List followers = cluster.getFollowers();
+        assertFalse(followers.isEmpty());
+        int success = this.sendTestTaskAndWait("follower apply ", followers.get(0), -1);
+
+        // elect new leader
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        LOG.info("Eelect new leader is {}", leader.getLeaderId());
+        // apply tasks to new leader
+        CountDownLatch latch = new CountDownLatch(10);
+        for (int i = 10; i < 20; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(latch));
+            leader.apply(task);
+        }
+        waitLatch(latch);
+
+        // restart old leader
+        LOG.info("restart old leader {}", oldLeader);
+        assertTrue(cluster.start(oldLeader.getEndpoint()));
+        // apply something
+        latch = new CountDownLatch(10);
+        for (int i = 20; i < 30; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(latch));
+            leader.apply(task);
+        }
+        waitLatch(latch);
+
+        // stop and clean old leader
+        cluster.stop(oldLeader.getEndpoint());
+        cluster.clean(oldLeader.getEndpoint());
+
+        // restart old leader
+        LOG.info("restart old leader {}", oldLeader);
+        assertTrue(cluster.start(oldLeader.getEndpoint()));
+        assertTrue(cluster.ensureSame(-1));
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(30 + success, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testJoinNodes() throws Exception {
+        final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final PeerId peer1 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 1);
+        final PeerId peer2 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 2);
+        final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+
+        final ArrayList peers = new ArrayList<>();
+        peers.add(peer0);
+
+        // start single cluster
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        assertTrue(cluster.start(peer0.getEndpoint()));
+
+        cluster.waitLeader();
+
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        Assert.assertEquals(leader.getNodeId().getPeerId(), peer0);
+        this.sendTestTaskAndWait(leader);
+
+        // start peer1
+        assertTrue(cluster.start(peer1.getEndpoint(), true, 300));
+        // add peer1
+        CountDownLatch latch = new CountDownLatch(1);
+        peers.add(peer1);
+        leader.addPeer(peer1, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        cluster.ensureSame(-1);
+        assertEquals(2, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(10, fsm.getLogs().size());
+        }
+
+        // add peer2 but not start
+        peers.add(peer2);
+        latch = new CountDownLatch(1);
+        leader.addPeer(peer2, new ExpectClosure(RaftError.ECATCHUP, latch));
+        waitLatch(latch);
+
+        // start peer2 after 2 seconds
+        Thread.sleep(2000);
+        assertTrue(cluster.start(peer2.getEndpoint(), true, 300));
+
+        Thread.sleep(10000);
+
+        // re-add peer2
+        latch = new CountDownLatch(2);
+        leader.addPeer(peer2, new ExpectClosure(latch));
+        // concurrent configuration change
+        leader.addPeer(peer3, new ExpectClosure(RaftError.EBUSY, latch));
+        waitLatch(latch);
+
+        // re-add peer2 directly
+
+        try {
+            leader.addPeer(peer2, new ExpectClosure(latch));
+            fail();
+        } catch (final IllegalArgumentException e) {
+            assertEquals("Peer already exists in current configuration", e.getMessage());
+        }
+
+        cluster.ensureSame();
+        assertEquals(3, cluster.getFsms().size());
+        assertEquals(2, cluster.getFollowers().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(10, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    private void waitLatch(final CountDownLatch latch) throws InterruptedException {
+        assertTrue(latch.await(30, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testRemoveFollower() throws Exception {
+        List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId followerPeer = followers.get(0).getNodeId().getPeerId();
+        final Endpoint followerAddr = followerPeer.getEndpoint();
+
+        // stop and clean follower
+        LOG.info("Stop and clean follower {}", followerPeer);
+        assertTrue(cluster.stop(followerAddr));
+        cluster.clean(followerAddr);
+
+        // remove follower
+        LOG.info("Remove follower {}", followerPeer);
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.removePeer(followerPeer, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+        followers = cluster.getFollowers();
+        assertEquals(1, followers.size());
+
+        peers = TestUtils.generatePeers(3);
+        assertTrue(peers.remove(followerPeer));
+
+        // start follower
+        LOG.info("Start and add follower {}", followerPeer);
+        assertTrue(cluster.start(followerAddr));
+        // re-add follower
+        latch = new CountDownLatch(1);
+        leader.addPeer(followerPeer, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        cluster.ensureSame();
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testRemoveLeader() throws Exception {
+        List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unittest", this.dataPath, peers);
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        // elect leader
+        cluster.waitLeader();
+
+        // get leader
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId oldLeader = leader.getNodeId().getPeerId().copy();
+        final Endpoint oldLeaderAddr = oldLeader.getEndpoint();
+
+        // remove old leader
+        LOG.info("Remove old leader {}", oldLeader);
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.removePeer(oldLeader, new ExpectClosure(latch));
+        waitLatch(latch);
+        Thread.sleep(100);
+
+        // elect new leader
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        LOG.info("New leader is {}", leader);
+        assertNotNull(leader);
+        // apply tasks to new leader
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        // stop and clean old leader
+        LOG.info("Stop and clean old leader {}", oldLeader);
+        assertTrue(cluster.stop(oldLeaderAddr));
+        cluster.clean(oldLeaderAddr);
+
+        // Add and start old leader
+        LOG.info("Start and add old leader {}", oldLeader);
+        assertTrue(cluster.start(oldLeaderAddr));
+
+        peers = TestUtils.generatePeers(3);
+        assertTrue(peers.remove(oldLeader));
+        latch = new CountDownLatch(1);
+        leader.addPeer(oldLeader, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+        cluster.ensureSame();
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testPreVote() throws Exception {
+        List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        Node leader = cluster.getLeader();
+        final long savedTerm = ((NodeImpl) leader).getCurrentTerm();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId followerPeer = followers.get(0).getNodeId().getPeerId();
+        final Endpoint followerAddr = followerPeer.getEndpoint();
+
+        // remove follower
+        LOG.info("Remove follower {}", followerPeer);
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.removePeer(followerPeer, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        Thread.sleep(2000);
+
+        // add follower
+        LOG.info("Add follower {}", followerAddr);
+        peers = TestUtils.generatePeers(3);
+        assertTrue(peers.remove(followerPeer));
+        latch = new CountDownLatch(1);
+        leader.addPeer(followerPeer, new ExpectClosure(latch));
+        waitLatch(latch);
+        leader = cluster.getLeader();
+        assertNotNull(leader);
+        // leader term should not be changed.
+        assertEquals(savedTerm, ((NodeImpl) leader).getCurrentTerm());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testSetPeer1() throws Exception {
+        final TestCluster cluster = new TestCluster("testSetPeer1", this.dataPath, new ArrayList<>());
+
+        final PeerId bootPeer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        assertTrue(cluster.start(bootPeer.getEndpoint()));
+        final List nodes = cluster.getFollowers();
+        assertEquals(1, nodes.size());
+
+        final List peers = new ArrayList<>();
+        peers.add(bootPeer);
+        // reset peers from empty
+        assertTrue(nodes.get(0).resetPeers(new Configuration(peers)).isOk());
+        cluster.waitLeader();
+        assertNotNull(cluster.getLeader());
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testSetPeer2() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId followerPeer1 = followers.get(0).getNodeId().getPeerId();
+        final Endpoint followerAddr1 = followerPeer1.getEndpoint();
+        final PeerId followerPeer2 = followers.get(1).getNodeId().getPeerId();
+        final Endpoint followerAddr2 = followerPeer2.getEndpoint();
+
+        LOG.info("Stop and clean follower {}", followerPeer1);
+        assertTrue(cluster.stop(followerAddr1));
+        cluster.clean(followerAddr1);
+
+        // apply tasks to leader again
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+        // set peer when no quorum die
+        final Endpoint leaderAddr = leader.getLeaderId().getEndpoint().copy();
+        LOG.info("Set peers to {}", leaderAddr);
+        final List newPeers = TestUtils.generatePeers(3);
+        assertTrue(newPeers.remove(followerPeer1));
+
+        LOG.info("Stop and clean follower {}", followerPeer2);
+        assertTrue(cluster.stop(followerAddr2));
+        cluster.clean(followerAddr2);
+
+        // leader will step-down, become follower
+        Thread.sleep(2000);
+        newPeers.clear();
+        newPeers.add(new PeerId(leaderAddr, 0));
+
+        // new peers equal to current conf
+        assertTrue(leader.resetPeers(new Configuration(peers)).isOk());
+        // set peer when quorum die
+        LOG.warn("Set peers to {}", leaderAddr);
+        assertTrue(leader.resetPeers(new Configuration(newPeers)).isOk());
+
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        assertNotNull(leader);
+        Assert.assertEquals(leaderAddr, leader.getNodeId().getPeerId().getEndpoint());
+
+        LOG.info("start follower {}", followerAddr1);
+        assertTrue(cluster.start(followerAddr1, true, 300));
+        LOG.info("start follower {}", followerAddr2);
+        assertTrue(cluster.start(followerAddr2, true, 300));
+
+        CountDownLatch latch = new CountDownLatch(1);
+        LOG.info("Add old follower {}", followerAddr1);
+        leader.addPeer(followerPeer1, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        latch = new CountDownLatch(1);
+        LOG.info("Add old follower {}", followerAddr2);
+        leader.addPeer(followerPeer2, new ExpectClosure(latch));
+        waitLatch(latch);
+
+        newPeers.add(followerPeer1);
+        newPeers.add(followerPeer2);
+
+        cluster.ensureSame();
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testRestoreSnasphot() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+        triggerLeaderSnapshot(cluster, leader);
+
+        // stop leader
+        final Endpoint leaderAddr = leader.getNodeId().getPeerId().getEndpoint().copy();
+        assertTrue(cluster.stop(leaderAddr));
+        Thread.sleep(2000);
+
+        // restart leader
+        cluster.waitLeader();
+        assertEquals(0, cluster.getLeaderFsm().getLoadSnapshotTimes());
+        assertTrue(cluster.start(leaderAddr));
+        cluster.ensureSame();
+        assertEquals(0, cluster.getLeaderFsm().getLoadSnapshotTimes());
+
+        cluster.stopAll();
+    }
+
+    private void triggerLeaderSnapshot(final TestCluster cluster, final Node leader) throws InterruptedException {
+        this.triggerLeaderSnapshot(cluster, leader, 1);
+    }
+
+    private void triggerLeaderSnapshot(final TestCluster cluster, final Node leader, final int times)
+                                                                                                     throws InterruptedException {
+        // trigger leader snapshot
+        // first snapshot will be triggered randomly
+        int snapshotTimes = cluster.getLeaderFsm().getSaveSnapshotTimes();
+        assertTrue("snapshotTimes=" + snapshotTimes + ", times=" + times, snapshotTimes == times - 1
+                                                                          || snapshotTimes == times);
+        final CountDownLatch latch = new CountDownLatch(1);
+        leader.snapshot(new ExpectClosure(latch));
+        waitLatch(latch);
+        assertEquals(snapshotTimes + 1, cluster.getLeaderFsm().getSaveSnapshotTimes());
+    }
+
+    @Test
+    public void testInstallSnapshotWithThrottle() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 200, false, new ThroughputSnapshotThrottle(1024, 1)));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        // stop follower1
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint();
+        assertTrue(cluster.stop(followerAddr));
+
+        cluster.waitLeader();
+
+        // apply something more
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        Thread.sleep(1000);
+
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader);
+        // apply something more
+        this.sendTestTaskAndWait(leader, 20, RaftError.SUCCESS);
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader, 2);
+
+        // wait leader to compact logs
+        Thread.sleep(1000);
+
+        // restart follower.
+        cluster.clean(followerAddr);
+        assertTrue(cluster.start(followerAddr, true, 300, false, new ThroughputSnapshotThrottle(1024, 1)));
+
+        Thread.sleep(2000);
+        cluster.ensureSame();
+
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(30, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testInstallLargeSnapshotWithThrottle() throws Exception {
+        final List peers = TestUtils.generatePeers(4);
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers.subList(0, 3));
+        for (int i = 0; i < peers.size() - 1; i++) {
+            final PeerId peer = peers.get(i);
+            final boolean started = cluster.start(peer.getEndpoint(), false, 200, false);
+            assertTrue(started);
+        }
+        cluster.waitLeader();
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        sendTestTaskAndWait(leader, 0, RaftError.SUCCESS);
+
+        cluster.ensureSame();
+
+        // apply something more
+        for (int i = 1; i < 100; i++) {
+            sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+        }
+
+        Thread.sleep(1000);
+
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader);
+
+        // apply something more
+        for (int i = 100; i < 200; i++) {
+            sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+        }
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader, 2);
+
+        // wait leader to compact logs
+        Thread.sleep(1000);
+
+        // add follower
+        final PeerId newPeer = peers.get(3);
+        final SnapshotThrottle snapshotThrottle = new ThroughputSnapshotThrottle(128, 1);
+        final boolean started = cluster.start(newPeer.getEndpoint(), true, 300, false, snapshotThrottle);
+        assertTrue(started);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        leader.addPeer(newPeer, status -> {
+            assertTrue(status.toString(), status.isOk());
+            latch.countDown();
+        });
+        waitLatch(latch);
+
+        cluster.ensureSame();
+
+        assertEquals(4, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(2000, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testInstallLargeSnapshot() throws Exception {
+        final List peers = TestUtils.generatePeers(4);
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers.subList(0, 3));
+        for (int i = 0; i < peers.size() - 1; i++) {
+            final PeerId peer = peers.get(i);
+            final boolean started = cluster.start(peer.getEndpoint(), false, 200, false);
+            assertTrue(started);
+        }
+        cluster.waitLeader();
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        sendTestTaskAndWait(leader, 0, RaftError.SUCCESS);
+
+        cluster.ensureSame();
+
+        // apply something more
+        for (int i = 1; i < 100; i++) {
+            sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+        }
+
+        Thread.sleep(1000);
+
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader);
+
+        // apply something more
+        for (int i = 100; i < 200; i++) {
+            sendTestTaskAndWait(leader, i * 10, RaftError.SUCCESS);
+        }
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader, 2);
+
+        // wait leader to compact logs
+        Thread.sleep(1000);
+
+        // add follower
+        final PeerId newPeer = peers.get(3);
+        final RaftOptions raftOptions = new RaftOptions();
+        raftOptions.setMaxByteCountPerRpc(128);
+        final boolean started = cluster.start(newPeer.getEndpoint(), true, 300, false, null, raftOptions);
+        assertTrue(started);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        leader.addPeer(newPeer, status -> {
+            assertTrue(status.toString(), status.isOk());
+            latch.countDown();
+        });
+        waitLatch(latch);
+
+        cluster.ensureSame();
+
+        assertEquals(4, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(2000, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testInstallSnapshot() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        // apply tasks to leader
+        this.sendTestTaskAndWait(leader);
+
+        cluster.ensureSame();
+
+        // stop follower1
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint();
+        assertTrue(cluster.stop(followerAddr));
+
+        // apply something more
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        // trigger leader snapshot
+        triggerLeaderSnapshot(cluster, leader);
+        // apply something more
+        this.sendTestTaskAndWait(leader, 20, RaftError.SUCCESS);
+        triggerLeaderSnapshot(cluster, leader, 2);
+
+        // wait leader to compact logs
+        Thread.sleep(50);
+
+        //restart follower.
+        cluster.clean(followerAddr);
+        assertTrue(cluster.start(followerAddr, true, 300));
+
+        Thread.sleep(2000);
+        cluster.ensureSame();
+
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(30, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testNoSnapshot() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+        final MockStateMachine fsm = new MockStateMachine(addr);
+        nodeOptions.setFsm(fsm);
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+        final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+        assertTrue(node.init(nodeOptions));
+        // wait node elect self as leader
+
+        Thread.sleep(2000);
+
+        this.sendTestTaskAndWait(node);
+
+        assertEquals(0, fsm.getSaveSnapshotTimes());
+        // do snapshot but returns error
+        CountDownLatch latch = new CountDownLatch(1);
+        node.snapshot(new ExpectClosure(RaftError.EINVAL, "Snapshot is not supported", latch));
+        waitLatch(latch);
+        assertEquals(0, fsm.getSaveSnapshotTimes());
+
+        latch = new CountDownLatch(1);
+        node.shutdown(new ExpectClosure(latch));
+        node.join();
+        waitLatch(latch);
+    }
+
+    @Test
+    public void testAutoSnapshot() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        NodeManager.getInstance().addAddress(addr);
+        final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+        final MockStateMachine fsm = new MockStateMachine(addr);
+        nodeOptions.setFsm(fsm);
+        nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+        nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOptions.setSnapshotIntervalSecs(10);
+        nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+        final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+        assertTrue(node.init(nodeOptions));
+        // wait node elect self as leader
+        Thread.sleep(2000);
+
+        sendTestTaskAndWait(node);
+
+        // wait for auto snapshot
+        Thread.sleep(10000);
+        // first snapshot will be triggered randomly
+        final int times = fsm.getSaveSnapshotTimes();
+        assertTrue("snapshotTimes=" + times, times >= 1);
+        assertTrue(fsm.getSnapshotIndex() > 0);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        node.shutdown(new ExpectClosure(latch));
+        node.join();
+        waitLatch(latch);
+    }
+
+    @Test
+    public void testLeaderShouldNotChange() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        // get leader
+        final Node leader0 = cluster.getLeader();
+        assertNotNull(leader0);
+        final long savedTerm = ((NodeImpl) leader0).getCurrentTerm();
+        LOG.info("Current leader is {}, term is {}", leader0, savedTerm);
+        Thread.sleep(5000);
+        cluster.waitLeader();
+        final Node leader1 = cluster.getLeader();
+        assertNotNull(leader1);
+        LOG.info("Current leader is {}", leader1);
+        assertEquals(savedTerm, ((NodeImpl) leader1).getCurrentTerm());
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testRecoverFollower() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+
+        Thread.sleep(100);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final Endpoint followerAddr = followers.get(0).getNodeId().getPeerId().getEndpoint().copy();
+        assertTrue(cluster.stop(followerAddr));
+
+        this.sendTestTaskAndWait(leader);
+
+        for (int i = 10; i < 30; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("no clusre" + i).getBytes());
+            final Task task = new Task(data, null);
+            leader.apply(task);
+        }
+        // wait leader to compact logs
+        Thread.sleep(5000);
+        // restart follower
+        assertTrue(cluster.start(followerAddr));
+        assertTrue(cluster.ensureSame(30));
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(30, fsm.getLogs().size());
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testLeaderTransfer() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        this.sendTestTaskAndWait(leader);
+
+        Thread.sleep(100);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+        LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+        assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+        Thread.sleep(1000);
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        Assert.assertEquals(leader.getNodeId().getPeerId(), targetPeer);
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testLeaderTransferBeforeLogIsCompleted() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 1));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+
+        Thread.sleep(100);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+        assertTrue(cluster.stop(targetPeer.getEndpoint()));
+        this.sendTestTaskAndWait(leader);
+        LOG.info("Transfer leadership from {} to {}", leader, targetPeer);
+        assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Task task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(RaftError.EBUSY, latch));
+        leader.apply(task);
+        waitLatch(latch);
+
+        assertTrue(cluster.start(targetPeer.getEndpoint()));
+        Thread.sleep(5000);
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        Assert.assertEquals(targetPeer, leader.getNodeId().getPeerId());
+        assertTrue(cluster.ensureSame(5));
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testLeaderTransferResumeOnFailure() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint(), false, 1));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+
+        final PeerId targetPeer = followers.get(0).getNodeId().getPeerId().copy();
+        assertTrue(cluster.stop(targetPeer.getEndpoint()));
+
+        this.sendTestTaskAndWait(leader);
+
+        assertTrue(leader.transferLeadershipTo(targetPeer).isOk());
+        final Node savedLeader = leader;
+        //try to apply task when transferring leadership
+        CountDownLatch latch = new CountDownLatch(1);
+        Task task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(RaftError.EBUSY, latch));
+        leader.apply(task);
+        waitLatch(latch);
+
+        Thread.sleep(100);
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        assertSame(leader, savedLeader);
+
+        // restart target peer
+        assertTrue(cluster.start(targetPeer.getEndpoint()));
+        Thread.sleep(100);
+        // retry apply task
+        latch = new CountDownLatch(1);
+        task = new Task(ByteBuffer.wrap("aaaaa".getBytes()), new ExpectClosure(latch));
+        leader.apply(task);
+        waitLatch(latch);
+
+        assertTrue(cluster.ensureSame(5));
+        cluster.stopAll();
+    }
+
+    /**
+     * mock state machine that fails to load snapshot.
+     * @author boyan (boyan@alibaba-inc.com)
+     *
+     * 2018-Apr-23 11:45:29 AM
+     */
+    static class MockFSM1 extends MockStateMachine {
+
+        public MockFSM1() {
+            this(new Endpoint(Utils.IP_ANY, 0));
+        }
+
+        public MockFSM1(final Endpoint address) {
+            super(address);
+        }
+
+        @Override
+        public boolean onSnapshotLoad(final SnapshotReader reader) {
+            return false;
+        }
+
+    }
+
+    @Test
+    public void testShutdownAndJoinWorkAfterInitFails() throws Exception {
+        final Endpoint addr = new Endpoint(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        NodeManager.getInstance().addAddress(addr);
+        {
+            final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+            final MockStateMachine fsm = new MockStateMachine(addr);
+            nodeOptions.setFsm(fsm);
+            nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+            nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+            nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+            nodeOptions.setSnapshotIntervalSecs(10);
+            nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+            final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+            assertTrue(node.init(nodeOptions));
+            Thread.sleep(1000);
+            this.sendTestTaskAndWait(node);
+
+            // save snapshot
+            final CountDownLatch latch = new CountDownLatch(1);
+            node.snapshot(new ExpectClosure(latch));
+            waitLatch(latch);
+            node.shutdown();
+            node.join();
+        }
+        {
+            final NodeOptions nodeOptions = createNodeOptionsWithSharedTimer();
+            final MockStateMachine fsm = new MockFSM1(addr);
+            nodeOptions.setFsm(fsm);
+            nodeOptions.setLogUri(this.dataPath + File.separator + "log");
+            nodeOptions.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+            nodeOptions.setRaftMetaUri(this.dataPath + File.separator + "meta");
+            nodeOptions.setSnapshotIntervalSecs(10);
+            nodeOptions.setInitialConf(new Configuration(Collections.singletonList(new PeerId(addr, 0))));
+
+            final Node node = new NodeImpl("unittest", new PeerId(addr, 0));
+            assertFalse(node.init(nodeOptions));
+            node.shutdown();
+            node.join();
+        }
+    }
+
+    @Test
+    public void testShuttingDownLeaderTriggerTimeoutNow() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        final Node oldLeader = leader;
+
+        LOG.info("Shutdown leader {}", leader);
+        leader.shutdown();
+        leader.join();
+
+        Thread.sleep(100);
+        leader = cluster.getLeader();
+        cluster.waitLeader();
+        assertNotNull(leader);
+        assertNotSame(leader, oldLeader);
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testRemovingLeaderTriggerTimeoutNow() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 300);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        final Node oldLeader = leader;
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        oldLeader.removePeer(oldLeader.getNodeId().getPeerId(), new ExpectClosure(latch));
+        waitLatch(latch);
+
+        Thread.sleep(100);
+        leader = cluster.getLeader();
+        assertNotNull(leader);
+        assertNotSame(leader, oldLeader);
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testTransferShouldWorkAfterInstallSnapshot() throws Exception {
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+        for (int i = 0; i < peers.size() - 1; i++) {
+            assertTrue(cluster.start(peers.get(i).getEndpoint()));
+        }
+
+        cluster.waitLeader();
+
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+
+        this.sendTestTaskAndWait(leader);
+
+        final List followers = cluster.getFollowers();
+        assertEquals(1, followers.size());
+
+        final PeerId follower = followers.get(0).getNodeId().getPeerId();
+        assertTrue(leader.transferLeadershipTo(follower).isOk());
+        Thread.sleep(2000);
+        leader = cluster.getLeader();
+        Assert.assertEquals(follower, leader.getNodeId().getPeerId());
+
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.snapshot(new ExpectClosure(latch));
+        waitLatch(latch);
+        latch = new CountDownLatch(1);
+        leader.snapshot(new ExpectClosure(latch));
+        waitLatch(latch);
+
+        // start the last peer which should be recover with snapshot.
+        final PeerId lastPeer = peers.get(2);
+        assertTrue(cluster.start(lastPeer.getEndpoint()));
+        Thread.sleep(5000);
+        assertTrue(leader.transferLeadershipTo(lastPeer).isOk());
+        Thread.sleep(2000);
+        leader = cluster.getLeader();
+        Assert.assertEquals(lastPeer, leader.getNodeId().getPeerId());
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(10, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testAppendEntriesWhenFollowerIsInErrorState() throws Exception {
+        // start five nodes
+        final List peers = TestUtils.generatePeers(5);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        cluster.waitLeader();
+        final Node oldLeader = cluster.getLeader();
+        assertNotNull(oldLeader);
+        // apply something
+        this.sendTestTaskAndWait(oldLeader);
+
+        // set one follower into error state
+        final List followers = cluster.getFollowers();
+        assertEquals(4, followers.size());
+        final Node errorNode = followers.get(0);
+        final PeerId errorPeer = errorNode.getNodeId().getPeerId().copy();
+        final Endpoint errorFollowerAddr = errorPeer.getEndpoint();
+        LOG.info("Set follower {} into error state", errorNode);
+        ((NodeImpl) errorNode).onError(new RaftException(EnumOutter.ErrorType.ERROR_TYPE_STATE_MACHINE, new Status(-1,
+            "Follower has something wrong.")));
+
+        // increase term  by stopping leader and electing a new leader again
+        final Endpoint oldLeaderAddr = oldLeader.getNodeId().getPeerId().getEndpoint().copy();
+        assertTrue(cluster.stop(oldLeaderAddr));
+        cluster.waitLeader();
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        LOG.info("Elect a new leader {}", leader);
+        // apply something again
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        // stop error follower
+        Thread.sleep(20);
+        LOG.info("Stop error follower {}", errorNode);
+        assertTrue(cluster.stop(errorFollowerAddr));
+        // restart error and old leader
+        LOG.info("Restart error follower {} and old leader {}", errorFollowerAddr, oldLeaderAddr);
+
+        assertTrue(cluster.start(errorFollowerAddr));
+        assertTrue(cluster.start(oldLeaderAddr));
+        cluster.ensureSame();
+        assertEquals(5, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testFollowerStartStopFollowing() throws Exception {
+        // start five nodes
+        final List peers = TestUtils.generatePeers(5);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+        cluster.waitLeader();
+        final Node firstLeader = cluster.getLeader();
+        assertNotNull(firstLeader);
+        // apply something
+        this.sendTestTaskAndWait(firstLeader);
+
+        // assert follow times
+        final List firstFollowers = cluster.getFollowers();
+        assertEquals(4, firstFollowers.size());
+        for (final Node node : firstFollowers) {
+            assertEquals(1, ((MockStateMachine) node.getOptions().getFsm()).getOnStartFollowingTimes());
+            assertEquals(0, ((MockStateMachine) node.getOptions().getFsm()).getOnStopFollowingTimes());
+        }
+
+        // stop leader and elect new one
+        final Endpoint fstLeaderAddr = firstLeader.getNodeId().getPeerId().getEndpoint();
+        assertTrue(cluster.stop(fstLeaderAddr));
+        cluster.waitLeader();
+        final Node secondLeader = cluster.getLeader();
+        assertNotNull(secondLeader);
+        this.sendTestTaskAndWait(secondLeader, 10, RaftError.SUCCESS);
+
+        // ensure start/stop following times
+        final List secondFollowers = cluster.getFollowers();
+        assertEquals(3, secondFollowers.size());
+        for (final Node node : secondFollowers) {
+            assertEquals(2, ((MockStateMachine) node.getOptions().getFsm()).getOnStartFollowingTimes());
+            assertEquals(1, ((MockStateMachine) node.getOptions().getFsm()).getOnStopFollowingTimes());
+        }
+
+        // transfer leadership to a follower
+        final PeerId targetPeer = secondFollowers.get(0).getNodeId().getPeerId().copy();
+        assertTrue(secondLeader.transferLeadershipTo(targetPeer).isOk());
+        Thread.sleep(100);
+        cluster.waitLeader();
+        final Node thirdLeader = cluster.getLeader();
+        Assert.assertEquals(targetPeer, thirdLeader.getNodeId().getPeerId());
+        this.sendTestTaskAndWait(thirdLeader, 20, RaftError.SUCCESS);
+
+        final List thirdFollowers = cluster.getFollowers();
+        assertEquals(3, thirdFollowers.size());
+        for (int i = 0; i < 3; i++) {
+            if (thirdFollowers.get(i).getNodeId().getPeerId().equals(secondLeader.getNodeId().getPeerId())) {
+                assertEquals(2,
+                    ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStartFollowingTimes());
+                assertEquals(1,
+                    ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStopFollowingTimes());
+                continue;
+            }
+            assertEquals(3, ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStartFollowingTimes());
+            assertEquals(2, ((MockStateMachine) thirdFollowers.get(i).getOptions().getFsm()).getOnStopFollowingTimes());
+        }
+
+        cluster.ensureSame();
+        cluster.stopAll();
+    }
+
+    @Test
+    public void readCommittedUserLog() throws Exception {
+        // setup cluster
+        final List peers = TestUtils.generatePeers(3);
+
+        final TestCluster cluster = new TestCluster("unitest", this.dataPath, peers, 1000);
+
+        for (final PeerId peer : peers) {
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+        cluster.waitLeader();
+
+        final Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        this.sendTestTaskAndWait(leader);
+
+        // index == 1 is a CONFIGURATION log, so real_index will be 2 when returned.
+        UserLog userLog = leader.readCommittedUserLog(1);
+        assertNotNull(userLog);
+        assertEquals(2, userLog.getIndex());
+        assertEquals("hello0", new String(userLog.getData().array()));
+
+        // index == 5 is a DATA log(a user log)
+        userLog = leader.readCommittedUserLog(5);
+        assertNotNull(userLog);
+        assertEquals(5, userLog.getIndex());
+        assertEquals("hello3", new String(userLog.getData().array()));
+
+        // index == 15 is greater than last_committed_index
+        try {
+            assertNull(leader.readCommittedUserLog(15));
+            fail();
+        } catch (final LogIndexOutOfBoundsException e) {
+            assertEquals(e.getMessage(), "Request index 15 is greater than lastAppliedIndex: 11");
+        }
+
+        // index == 0 invalid request
+        try {
+            assertNull(leader.readCommittedUserLog(0));
+            fail();
+        } catch (final LogIndexOutOfBoundsException e) {
+            assertEquals(e.getMessage(), "Request index is invalid: 0");
+        }
+        LOG.info("Trigger leader snapshot");
+        CountDownLatch latch = new CountDownLatch(1);
+        leader.snapshot(new ExpectClosure(latch));
+        waitLatch(latch);
+
+        // remove and add a peer to add two CONFIGURATION logs
+        final List followers = cluster.getFollowers();
+        assertEquals(2, followers.size());
+        final Node testFollower = followers.get(0);
+        latch = new CountDownLatch(1);
+        leader.removePeer(testFollower.getNodeId().getPeerId(), new ExpectClosure(latch));
+        waitLatch(latch);
+        latch = new CountDownLatch(1);
+        leader.addPeer(testFollower.getNodeId().getPeerId(), new ExpectClosure(latch));
+        waitLatch(latch);
+
+        this.sendTestTaskAndWait(leader, 10, RaftError.SUCCESS);
+
+        // trigger leader snapshot for the second time, after this the log of index 1~11 will be deleted.
+        LOG.info("Trigger leader snapshot");
+        latch = new CountDownLatch(1);
+        leader.snapshot(new ExpectClosure(latch));
+        waitLatch(latch);
+        Thread.sleep(100);
+
+        // index == 5 log has been deleted in log_storage.
+        try {
+            leader.readCommittedUserLog(5);
+            fail();
+        } catch (final LogNotFoundException e) {
+            assertEquals("User log is deleted at index: 5", e.getMessage());
+        }
+
+        // index == 12、index == 13、index=14、index=15 are 4 CONFIGURATION logs(joint consensus), so real_index will be 16 when returned.
+        userLog = leader.readCommittedUserLog(12);
+        assertNotNull(userLog);
+        assertEquals(16, userLog.getIndex());
+        assertEquals("hello10", new String(userLog.getData().array()));
+
+        // now index == 17 is a user log
+        userLog = leader.readCommittedUserLog(17);
+        assertNotNull(userLog);
+        assertEquals(17, userLog.getIndex());
+        assertEquals("hello11", new String(userLog.getData().array()));
+
+        cluster.ensureSame();
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+            for (int i = 0; i < 20; i++) {
+                assertEquals("hello" + i, new String(fsm.getLogs().get(i).array()));
+            }
+        }
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testBootStrapWithSnapshot() throws Exception {
+        final Endpoint addr = JRaftUtils.getEndPoint("127.0.0.1:5006");
+        final MockStateMachine fsm = new MockStateMachine(addr);
+
+        for (char ch = 'a'; ch <= 'z'; ch++) {
+            fsm.getLogs().add(ByteBuffer.wrap(new byte[] { (byte) ch }));
+        }
+
+        final BootstrapOptions opts = new BootstrapOptions();
+        opts.setLastLogIndex(fsm.getLogs().size());
+        opts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        opts.setLogUri(this.dataPath + File.separator + "log");
+        opts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        opts.setGroupConf(JRaftUtils.getConfiguration("127.0.0.1:5006"));
+        opts.setFsm(fsm);
+
+        NodeManager.getInstance().addAddress(addr);
+        assertTrue(JRaftUtils.bootstrap(opts));
+
+        final NodeOptions nodeOpts = createNodeOptionsWithSharedTimer();
+        nodeOpts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOpts.setLogUri(this.dataPath + File.separator + "log");
+        nodeOpts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOpts.setFsm(fsm);
+
+        final NodeImpl node = new NodeImpl("test", new PeerId(addr, 0));
+        assertTrue(node.init(nodeOpts));
+        assertEquals(26, fsm.getLogs().size());
+
+        for (int i = 0; i < 26; i++) {
+            assertEquals('a' + i, fsm.getLogs().get(i).get());
+        }
+
+        while (!node.isLeader()) {
+            Thread.sleep(20);
+        }
+        this.sendTestTaskAndWait(node);
+        assertEquals(36, fsm.getLogs().size());
+        node.shutdown();
+        node.join();
+    }
+
+    @Test
+    public void testBootStrapWithoutSnapshot() throws Exception {
+        final Endpoint addr = JRaftUtils.getEndPoint("127.0.0.1:5006");
+        final MockStateMachine fsm = new MockStateMachine(addr);
+
+        final BootstrapOptions opts = new BootstrapOptions();
+        opts.setLastLogIndex(0);
+        opts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        opts.setLogUri(this.dataPath + File.separator + "log");
+        opts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        opts.setGroupConf(JRaftUtils.getConfiguration("127.0.0.1:5006"));
+        opts.setFsm(fsm);
+
+        NodeManager.getInstance().addAddress(addr);
+        assertTrue(JRaftUtils.bootstrap(opts));
+
+        final NodeOptions nodeOpts = createNodeOptionsWithSharedTimer();
+        nodeOpts.setRaftMetaUri(this.dataPath + File.separator + "meta");
+        nodeOpts.setLogUri(this.dataPath + File.separator + "log");
+        nodeOpts.setSnapshotUri(this.dataPath + File.separator + "snapshot");
+        nodeOpts.setFsm(fsm);
+
+        final NodeImpl node = new NodeImpl("test", new PeerId(addr, 0));
+        assertTrue(node.init(nodeOpts));
+        while (!node.isLeader()) {
+            Thread.sleep(20);
+        }
+        this.sendTestTaskAndWait(node);
+        assertEquals(10, fsm.getLogs().size());
+        node.shutdown();
+        node.join();
+    }
+
+    @Test
+    public void testChangePeers() throws Exception {
+        final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final TestCluster cluster = new TestCluster("testChangePeers", this.dataPath, Collections.singletonList(peer0));
+        assertTrue(cluster.start(peer0.getEndpoint()));
+
+        cluster.waitLeader();
+        Node leader = cluster.getLeader();
+        this.sendTestTaskAndWait(leader);
+
+        for (int i = 1; i < 10; i++) {
+            final PeerId peer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + i);
+            assertTrue(cluster.start(peer.getEndpoint(), true, 300));
+        }
+        for (int i = 0; i < 9; i++) {
+            cluster.waitLeader();
+            leader = cluster.getLeader();
+            assertNotNull(leader);
+            PeerId peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + i);
+            Assert.assertEquals(peer, leader.getNodeId().getPeerId());
+            peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + i + 1);
+            final SynchronizedClosure done = new SynchronizedClosure();
+            leader.changePeers(new Configuration(Collections.singletonList(peer)), done);
+            assertTrue(done.await().isOk());
+        }
+        assertTrue(cluster.ensureSame());
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testChangePeersAddMultiNodes() throws Exception {
+        final PeerId peer0 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT);
+        final TestCluster cluster = new TestCluster("testChangePeersAddMultiNodes", this.dataPath,
+            Collections.singletonList(peer0));
+        assertTrue(cluster.start(peer0.getEndpoint()));
+
+        cluster.waitLeader();
+        final Node leader = cluster.getLeader();
+        this.sendTestTaskAndWait(leader);
+
+        final Configuration conf = new Configuration();
+        for (int i = 0; i < 3; i++) {
+            final PeerId peer = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + i);
+            conf.addPeer(peer);
+        }
+
+        PeerId peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + 1);
+        // fail, because the peers are not started.
+        final SynchronizedClosure done = new SynchronizedClosure();
+        leader.changePeers(new Configuration(Collections.singletonList(peer)), done);
+        Assert.assertEquals(RaftError.ECATCHUP, done.await().getRaftError());
+
+        // start peer1
+        assertTrue(cluster.start(peer.getEndpoint()));
+        // still fail, because peer2 is not started
+        done.reset();
+        leader.changePeers(conf, done);
+        Assert.assertEquals(RaftError.ECATCHUP, done.await().getRaftError());
+        // start peer2
+        peer = new PeerId(TestUtils.getMyIp(), peer0.getEndpoint().getPort() + 2);
+        assertTrue(cluster.start(peer.getEndpoint()));
+        done.reset();
+        // works
+        leader.changePeers(conf, done);
+        assertTrue(done.await().isOk());
+
+        assertTrue(cluster.ensureSame());
+        assertEquals(3, cluster.getFsms().size());
+        for (final MockStateMachine fsm : cluster.getFsms()) {
+            assertEquals(10, fsm.getLogs().size());
+        }
+
+        cluster.stopAll();
+    }
+
+    @Test
+    public void testChangePeersStepsDownInJointConsensus() throws Exception {
+        final List peers = new ArrayList<>();
+        final PeerId peer0 = JRaftUtils.getPeerId("127.0.0.1:5006");
+        final PeerId peer1 = JRaftUtils.getPeerId("127.0.0.1:5007");
+        final PeerId peer2 = JRaftUtils.getPeerId("127.0.0.1:5008");
+        final PeerId peer3 = JRaftUtils.getPeerId("127.0.0.1:5009");
+
+        // start single cluster
+        peers.add(peer0);
+        final TestCluster cluster = new TestCluster("testChangePeersStepsDownInJointConsensus", this.dataPath, peers);
+        assertTrue(cluster.start(peer0.getEndpoint()));
+
+        cluster.waitLeader();
+        Node leader = cluster.getLeader();
+        assertNotNull(leader);
+        this.sendTestTaskAndWait(leader);
+
+        // start peer1-3
+        assertTrue(cluster.start(peer1.getEndpoint()));
+        assertTrue(cluster.start(peer2.getEndpoint()));
+        assertTrue(cluster.start(peer3.getEndpoint()));
+
+        final Configuration conf = new Configuration();
+        conf.addPeer(peer0);
+        conf.addPeer(peer1);
+        conf.addPeer(peer2);
+        conf.addPeer(peer3);
+
+        // change peers
+        final SynchronizedClosure done = new SynchronizedClosure();
+        leader.changePeers(conf, done);
+        assertTrue(done.await().isOk());
+
+        // stop peer3
+        assertTrue(cluster.stop(peer3.getEndpoint()));
+
+        conf.removePeer(peer0);
+        conf.removePeer(peer1);
+
+        // Change peers to [peer2, peer3], which must fail since peer3 is stopped
+        done.reset();
+        leader.changePeers(conf, done);
+        Assert.assertEquals(RaftError.EPERM, done.await().getRaftError());
+        LOG.info(done.getStatus().toString());
+
+        assertFalse(((NodeImpl) leader).getConf().isStable());
+
+        leader = cluster.getLeader();
+        assertNull(leader);
+
+        assertTrue(cluster.start(peer3.getEndpoint()));
+        Thread.sleep(1000);
+        cluster.waitLeader();
+        leader = cluster.getLeader();
+        final List thePeers = leader.listPeers();
+        assertTrue(thePeers.size() > 0);
+        assertEquals(conf.getPeerSet(), new HashSet<>(thePeers));
+
+        cluster.stopAll();
+    }
+
+    static class ChangeArg {
+        TestCluster      c;
+        List     peers;
+        volatile boolean stop;
+        boolean          dontRemoveFirstPeer;
+
+        public ChangeArg(final TestCluster c, final List peers, final boolean stop,
+                         final boolean dontRemoveFirstPeer) {
+            super();
+            this.c = c;
+            this.peers = peers;
+            this.stop = stop;
+            this.dontRemoveFirstPeer = dontRemoveFirstPeer;
+        }
+
+    }
+
+    private Future startChangePeersThread(final ChangeArg arg) {
+
+        final Set expectedErrors = new HashSet<>();
+        expectedErrors.add(RaftError.EBUSY);
+        expectedErrors.add(RaftError.EPERM);
+        expectedErrors.add(RaftError.ECATCHUP);
+
+        return Utils.runInThread(() -> {
+            try {
+                while (!arg.stop) {
+                    arg.c.waitLeader();
+                    final Node leader = arg.c.getLeader();
+                    if (leader == null) {
+                        continue;
+                    }
+                    // select peers in random
+                    final Configuration conf = new Configuration();
+                    if (arg.dontRemoveFirstPeer) {
+                        conf.addPeer(arg.peers.get(0));
+                    }
+                    for (int i = 0; i < arg.peers.size(); i++) {
+                        final boolean select = ThreadLocalRandom.current().nextInt(64) < 32;
+                        if (select && !conf.contains(arg.peers.get(i))) {
+                            conf.addPeer(arg.peers.get(i));
+                        }
+                    }
+                    if (conf.isEmpty()) {
+                        LOG.warn("No peer has been selected");
+                        continue;
+                    }
+                    final SynchronizedClosure done = new SynchronizedClosure();
+                    leader.changePeers(conf, done);
+                    done.await();
+                    assertTrue(done.getStatus().toString(),
+                        done.getStatus().isOk() || expectedErrors.contains(done.getStatus().getRaftError()));
+                }
+            } catch (final InterruptedException e) {
+                LOG.error("ChangePeersThread is interrupted", e);
+            }
+        });
+    }
+
+    @Test
+    public void testChangePeersChaosWithSnapshot() throws Exception {
+        // start cluster
+        final List peers = new ArrayList<>();
+        peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+        final TestCluster cluster = new TestCluster("testChangePeersChaosWithSnapshot", this.dataPath, peers, 1000);
+        assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 2));
+        // start other peers
+        for (int i = 1; i < 10; i++) {
+            final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+            peers.add(peer);
+            assertTrue(cluster.start(peer.getEndpoint()));
+        }
+
+        final ChangeArg arg = new ChangeArg(cluster, peers, false, false);
+
+        final Future future = startChangePeersThread(arg);
+        for (int i = 0; i < 5000;) {
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            if (leader == null) {
+                continue;
+            }
+            final SynchronizedClosure done = new SynchronizedClosure();
+            final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+            leader.apply(task);
+            final Status status = done.await();
+            if (status.isOk()) {
+                if (++i % 100 == 0) {
+                    System.out.println("Progress:" + i);
+                }
+            } else {
+                assertEquals(RaftError.EPERM, status.getRaftError());
+            }
+        }
+        arg.stop = true;
+        future.get();
+        cluster.waitLeader();
+        final SynchronizedClosure done = new SynchronizedClosure();
+        final Node leader = cluster.getLeader();
+        leader.changePeers(new Configuration(peers), done);
+        final Status st = done.await();
+        assertTrue(st.getErrorMsg(), st.isOk());
+        cluster.ensureSame();
+        assertEquals(10, cluster.getFsms().size());
+        try {
+            for (final MockStateMachine fsm : cluster.getFsms()) {
+                assertTrue(fsm.getLogs().size() >= 5000);
+            }
+        } finally {
+            cluster.stopAll();
+        }
+    }
+
+    @Test
+    public void testChangePeersChaosWithoutSnapshot() throws Exception {
+        // start cluster
+        final List peers = new ArrayList<>();
+        peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+        final TestCluster cluster = new TestCluster("testChangePeersChaosWithoutSnapshot", this.dataPath, peers, 1000);
+        assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 100000));
+        // start other peers
+        for (int i = 1; i < 10; i++) {
+            final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+            peers.add(peer);
+            assertTrue(cluster.start(peer.getEndpoint(), true, 10000));
+        }
+
+        final ChangeArg arg = new ChangeArg(cluster, peers, false, true);
+
+        final Future future = startChangePeersThread(arg);
+        final int tasks = 5000;
+        for (int i = 0; i < tasks;) {
+            cluster.waitLeader();
+            final Node leader = cluster.getLeader();
+            if (leader == null) {
+                continue;
+            }
+            final SynchronizedClosure done = new SynchronizedClosure();
+            final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+            leader.apply(task);
+            final Status status = done.await();
+            if (status.isOk()) {
+                if (++i % 100 == 0) {
+                    System.out.println("Progress:" + i);
+                }
+            } else {
+                assertEquals(RaftError.EPERM, status.getRaftError());
+            }
+        }
+        arg.stop = true;
+        future.get();
+        cluster.waitLeader();
+        final SynchronizedClosure done = new SynchronizedClosure();
+        final Node leader = cluster.getLeader();
+        leader.changePeers(new Configuration(peers), done);
+        assertTrue(done.await().isOk());
+        cluster.ensureSame();
+        assertEquals(10, cluster.getFsms().size());
+        try {
+            for (final MockStateMachine fsm : cluster.getFsms()) {
+                final int logSize = fsm.getLogs().size();
+                assertTrue("logSize=" + logSize, logSize >= tasks);
+            }
+        } finally {
+            cluster.stopAll();
+        }
+    }
+
+    @Test
+    public void testChangePeersChaosApplyTasks() throws Exception {
+        // start cluster
+        final List peers = new ArrayList<>();
+        peers.add(new PeerId("127.0.0.1", TestUtils.INIT_PORT));
+        final TestCluster cluster = new TestCluster("testChangePeersChaosApplyTasks", this.dataPath, peers, 1000);
+        assertTrue(cluster.start(peers.get(0).getEndpoint(), false, 100000));
+        // start other peers
+        for (int i = 1; i < 10; i++) {
+            final PeerId peer = new PeerId("127.0.0.1", TestUtils.INIT_PORT + i);
+            peers.add(peer);
+            assertTrue(cluster.start(peer.getEndpoint(), true, 100000));
+        }
+
+        final int threads = 3;
+        final List args = new ArrayList<>();
+        final List> futures = new ArrayList<>();
+        final CountDownLatch latch = new CountDownLatch(threads);
+        for (int t = 0; t < threads; t++) {
+            final ChangeArg arg = new ChangeArg(cluster, peers, false, true);
+            args.add(arg);
+            futures.add(startChangePeersThread(arg));
+
+            Utils.runInThread(() -> {
+                try {
+                    for (int i = 0; i < 5000;) {
+                        cluster.waitLeader();
+                        final Node leader = cluster.getLeader();
+                        if (leader == null) {
+                            continue;
+                        }
+                        final SynchronizedClosure done = new SynchronizedClosure();
+                        final Task task = new Task(ByteBuffer.wrap(("hello" + i).getBytes()), done);
+                        leader.apply(task);
+                        final Status status = done.await();
+                        if (status.isOk()) {
+                            if (++i % 100 == 0) {
+                                System.out.println("Progress:" + i);
+                            }
+                        } else {
+                            assertEquals(RaftError.EPERM, status.getRaftError());
+                        }
+                    }
+                } catch (final Exception e) {
+                    e.printStackTrace();
+                } finally {
+                    latch.countDown();
+                }
+            });
+        }
+
+        latch.await();
+        for (final ChangeArg arg : args) {
+            arg.stop = true;
+        }
+        for (final Future future : futures) {
+            future.get();
+        }
+
+        cluster.waitLeader();
+        final SynchronizedClosure done = new SynchronizedClosure();
+        final Node leader = cluster.getLeader();
+        leader.changePeers(new Configuration(peers), done);
+        assertTrue(done.await().isOk());
+        cluster.ensureSame();
+        assertEquals(10, cluster.getFsms().size());
+        try {
+            for (final MockStateMachine fsm : cluster.getFsms()) {
+                final int logSize = fsm.getLogs().size();
+                assertTrue("logSize= " + logSize, logSize >= 5000 * threads);
+            }
+        } finally {
+            cluster.stopAll();
+        }
+    }
+
+    private NodeOptions createNodeOptionsWithSharedTimer() {
+        final NodeOptions options = new NodeOptions();
+        options.setSharedElectionTimer(true);
+        options.setSharedVoteTimer(true);
+        return options;
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java
new file mode 100644
index 0000000..e05693b
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReadOnlyServiceTest.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.ReadIndexState;
+import com.alipay.sofa.jraft.entity.ReadIndexStatus;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReadOnlyServiceOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexResponse;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosure;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Bytes;
+import com.alipay.sofa.jraft.util.Utils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(MockitoJUnitRunner.class)
+public class ReadOnlyServiceTest {
+
+    private ReadOnlyServiceImpl readOnlyServiceImpl;
+
+    @Mock
+    private NodeImpl            node;
+
+    @Mock
+    private FSMCaller           fsmCaller;
+
+    @Before
+    public void setup() {
+        this.readOnlyServiceImpl = new ReadOnlyServiceImpl();
+        final ReadOnlyServiceOptions opts = new ReadOnlyServiceOptions();
+        opts.setFsmCaller(this.fsmCaller);
+        opts.setNode(this.node);
+        opts.setRaftOptions(new RaftOptions());
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+        Mockito.when(this.node.getOptions()).thenReturn(new NodeOptions());
+        Mockito.when(this.node.getGroupId()).thenReturn("test");
+        Mockito.when(this.node.getServerId()).thenReturn(new PeerId("localhost:8081", 0));
+        assertTrue(this.readOnlyServiceImpl.init(opts));
+    }
+
+    @After
+    public void teardown() throws Exception {
+        this.readOnlyServiceImpl.shutdown();
+        this.readOnlyServiceImpl.join();
+    }
+
+    @Test
+    public void testAddRequest() throws Exception {
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+
+            }
+        });
+        this.readOnlyServiceImpl.flush();
+        Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher() {
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof ReadIndexRequest) {
+                    final ReadIndexRequest req = (ReadIndexRequest) argument;
+                    return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+                           && req.getEntriesCount() == 1
+                           && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+                }
+                return false;
+            }
+
+        }), Mockito.any());
+    }
+
+    @Test
+    public void testAddRequestOnResponsePending() throws Exception {
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertTrue(status.isOk());
+                assertEquals(index, 1);
+                assertArrayEquals(reqCtx, requestContext);
+                latch.countDown();
+            }
+        });
+        this.readOnlyServiceImpl.flush();
+
+        final ArgumentCaptor closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+        Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher() {
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof ReadIndexRequest) {
+                    final ReadIndexRequest req = (ReadIndexRequest) argument;
+                    return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+                           && req.getEntriesCount() == 1
+                           && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+                }
+                return false;
+            }
+
+        }), closureCaptor.capture());
+
+        final RpcResponseClosure closure = closureCaptor.getValue();
+
+        assertNotNull(closure);
+
+        closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+        assertTrue(this.readOnlyServiceImpl.getPendingNotifyStatus().isEmpty());
+        closure.run(Status.OK());
+        assertEquals(this.readOnlyServiceImpl.getPendingNotifyStatus().size(), 1);
+        this.readOnlyServiceImpl.onApplied(2);
+        latch.await();
+    }
+
+    @Test
+    public void testAddRequestOnResponseFailure() throws Exception {
+        Mockito.when(this.fsmCaller.getLastAppliedIndex()).thenReturn(2L);
+
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertFalse(status.isOk());
+                assertEquals(index, -1);
+                assertArrayEquals(reqCtx, requestContext);
+                latch.countDown();
+            }
+        });
+        this.readOnlyServiceImpl.flush();
+
+        final ArgumentCaptor closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+        Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher() {
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof ReadIndexRequest) {
+                    final ReadIndexRequest req = (ReadIndexRequest) argument;
+                    return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+                           && req.getEntriesCount() == 1
+                           && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+                }
+                return false;
+            }
+
+        }), closureCaptor.capture());
+
+        final RpcResponseClosure closure = closureCaptor.getValue();
+
+        assertNotNull(closure);
+
+        closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+        closure.run(new Status(-1, "test"));
+        latch.await();
+    }
+
+    @Test
+    public void testAddRequestOnResponseSuccess() throws Exception {
+
+        Mockito.when(this.fsmCaller.getLastAppliedIndex()).thenReturn(2L);
+
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertTrue(status.isOk());
+                assertEquals(index, 1);
+                assertArrayEquals(reqCtx, requestContext);
+                latch.countDown();
+            }
+        });
+        this.readOnlyServiceImpl.flush();
+
+        final ArgumentCaptor closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+        Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher() {
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof ReadIndexRequest) {
+                    final ReadIndexRequest req = (ReadIndexRequest) argument;
+                    return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+                           && req.getEntriesCount() == 1
+                           && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+                }
+                return false;
+            }
+
+        }), closureCaptor.capture());
+
+        final RpcResponseClosure closure = closureCaptor.getValue();
+
+        assertNotNull(closure);
+
+        closure.setResponse(ReadIndexResponse.newBuilder().setIndex(1).setSuccess(true).build());
+        closure.run(Status.OK());
+        latch.await();
+    }
+
+    @Test
+    public void testOnApplied() throws Exception {
+        final ArrayList states = new ArrayList<>();
+        final byte[] reqContext = TestUtils.getRandomBytes();
+        final CountDownLatch latch = new CountDownLatch(1);
+        final ReadIndexState state = new ReadIndexState(new Bytes(reqContext), new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertTrue(status.isOk());
+                assertEquals(index, 1);
+                assertArrayEquals(reqCtx, reqContext);
+                latch.countDown();
+            }
+        }, Utils.monotonicMs());
+        state.setIndex(1);
+        states.add(state);
+        final ReadIndexStatus readIndexStatus = new ReadIndexStatus(states, null, 1);
+        this.readOnlyServiceImpl.getPendingNotifyStatus().put(1L, Arrays.asList(readIndexStatus));
+
+        this.readOnlyServiceImpl.onApplied(2);
+        latch.await();
+        assertTrue(this.readOnlyServiceImpl.getPendingNotifyStatus().isEmpty());
+    }
+
+    @Test
+    public void testOverMaxReadIndexLag() throws Exception {
+        Mockito.when(this.fsmCaller.getLastAppliedIndex()).thenReturn(1L);
+        this.readOnlyServiceImpl.getRaftOptions().setMaxReadIndexLag(50);
+
+        final byte[] requestContext = TestUtils.getRandomBytes();
+        final CountDownLatch latch = new CountDownLatch(1);
+        final String errMsg = "Fail to run ReadIndex task, the gap of current node's apply index between leader's commit index over maxReadIndexLag";
+        this.readOnlyServiceImpl.addRequest(requestContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                assertFalse(status.isOk());
+                assertEquals(status.getErrorMsg(), errMsg);
+                assertEquals(index, -1);
+                assertArrayEquals(reqCtx, requestContext);
+                latch.countDown();
+            }
+        });
+        this.readOnlyServiceImpl.flush();
+
+        final ArgumentCaptor closureCaptor = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+        Mockito.verify(this.node).handleReadIndexRequest(Mockito.argThat(new ArgumentMatcher() {
+
+            @Override
+            public boolean matches(final Object argument) {
+                if (argument instanceof ReadIndexRequest) {
+                    final ReadIndexRequest req = (ReadIndexRequest) argument;
+                    return req.getGroupId().equals("test") && req.getServerId().equals("localhost:8081:0")
+                           && req.getEntriesCount() == 1
+                           && Arrays.equals(requestContext, req.getEntries(0).toByteArray());
+                }
+                return false;
+            }
+
+        }), closureCaptor.capture());
+
+        final RpcResponseClosure closure = closureCaptor.getValue();
+
+        assertNotNull(closure);
+
+        closure.setResponse(ReadIndexResponse.newBuilder().setIndex(52).setSuccess(true).build());
+        closure.run(Status.OK());
+        latch.await();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java
new file mode 100644
index 0000000..4b1c4be
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorGroupTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReplicatorGroupOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.google.protobuf.ByteString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class ReplicatorGroupTest {
+
+    static final Logger         LOG            = LoggerFactory.getLogger(ReplicatorGroupTest.class);
+
+    private TimerManager        timerManager;
+    private ReplicatorGroupImpl replicatorGroup;
+    @Mock
+    private BallotBox           ballotBox;
+    @Mock
+    private LogManager          logManager;
+    @Mock
+    private NodeImpl            node;
+    @Mock
+    private RaftClientService   rpcService;
+    @Mock
+    private SnapshotStorage     snapshotStorage;
+    private final NodeOptions   options        = new NodeOptions();
+    private final RaftOptions   raftOptions    = new RaftOptions();
+    private final PeerId        peerId1        = new PeerId("localhost", 8082);
+    private final PeerId        peerId2        = new PeerId("localhost", 8083);
+    private final PeerId        peerId3        = new PeerId("localhost", 8084);
+    private final AtomicInteger errorCounter   = new AtomicInteger(0);
+    private final AtomicInteger stoppedCounter = new AtomicInteger(0);
+    private final AtomicInteger startedCounter = new AtomicInteger(0);
+
+    @Before
+    public void setup() {
+        this.timerManager = new TimerManager(5);
+        this.replicatorGroup = new ReplicatorGroupImpl();
+        final ReplicatorGroupOptions rgOpts = new ReplicatorGroupOptions();
+        rgOpts.setHeartbeatTimeoutMs(heartbeatTimeout(this.options.getElectionTimeoutMs()));
+        rgOpts.setElectionTimeoutMs(this.options.getElectionTimeoutMs());
+        rgOpts.setLogManager(this.logManager);
+        rgOpts.setBallotBox(this.ballotBox);
+        rgOpts.setNode(this.node);
+        rgOpts.setRaftRpcClientService(this.rpcService);
+        rgOpts.setSnapshotStorage(this.snapshotStorage);
+        rgOpts.setRaftOptions(this.raftOptions);
+        rgOpts.setTimerManager(this.timerManager);
+        Mockito.when(this.logManager.getLastLogIndex()).thenReturn(10L);
+        Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(false));
+        Mockito.when(this.node.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8081)));
+        mockSendEmptyEntries();
+        assertTrue(this.replicatorGroup.init(this.node.getNodeId(), rgOpts));
+    }
+
+    @Test
+    public void testAddReplicatorAndFailed() {
+        this.replicatorGroup.resetTerm(1);
+        assertFalse(this.replicatorGroup.addReplicator(this.peerId1));
+        assertEquals(this.replicatorGroup.getFailureReplicators().get(this.peerId1), ReplicatorType.Follower);
+    }
+
+    @Test
+    public void testAddLearnerFailure() {
+        this.replicatorGroup.resetTerm(1);
+        assertFalse(this.replicatorGroup.addReplicator(this.peerId1, ReplicatorType.Learner));
+        assertEquals(this.replicatorGroup.getFailureReplicators().get(this.peerId1), ReplicatorType.Learner);
+    }
+
+    @Test
+    public void testAddLearnerSuccess() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        assertTrue(this.replicatorGroup.addReplicator(this.peerId1, ReplicatorType.Learner));
+        assertNotNull(this.replicatorGroup.getReplicatorMap().get(this.peerId1));
+        assertNull(this.replicatorGroup.getFailureReplicators().get(this.peerId1));
+    }
+
+    @Test
+    public void testAddReplicatorSuccess() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        assertTrue(this.replicatorGroup.addReplicator(this.peerId1));
+        assertNull(this.replicatorGroup.getFailureReplicators().get(this.peerId1));
+    }
+
+    @Test
+    public void testStopReplicator() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(this.peerId1);
+        assertTrue(this.replicatorGroup.stopReplicator(this.peerId1));
+    }
+
+    @Test
+    public void testStopAllReplicator() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(this.peerId1);
+        this.replicatorGroup.addReplicator(this.peerId2);
+        this.replicatorGroup.addReplicator(this.peerId3);
+        assertTrue(this.replicatorGroup.contains(this.peerId1));
+        assertTrue(this.replicatorGroup.contains(this.peerId2));
+        assertTrue(this.replicatorGroup.contains(this.peerId3));
+        assertTrue(this.replicatorGroup.stopAll());
+    }
+
+    @Test
+    public void testReplicatorWithNoRepliactorStateListener() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(this.peerId1);
+        this.replicatorGroup.addReplicator(this.peerId2);
+        this.replicatorGroup.addReplicator(this.peerId3);
+        assertTrue(this.replicatorGroup.stopAll());
+        assertEquals(0, this.startedCounter.get());
+        assertEquals(0, this.errorCounter.get());
+        assertEquals(0, this.stoppedCounter.get());
+
+    }
+
+    class UserReplicatorStateListener implements Replicator.ReplicatorStateListener {
+        @Override
+        public void onCreated(final PeerId peer) {
+            LOG.info("Replicator has created");
+            ReplicatorGroupTest.this.startedCounter.incrementAndGet();
+        }
+
+        @Override
+        public void onError(final PeerId peer, final Status status) {
+            LOG.info("Replicator has errors");
+            ReplicatorGroupTest.this.errorCounter.incrementAndGet();
+        }
+
+        @Override
+        public void onDestroyed(final PeerId peer) {
+            LOG.info("Replicator has been destroyed");
+            ReplicatorGroupTest.this.stoppedCounter.incrementAndGet();
+        }
+    }
+
+    @Test
+    public void testTransferLeadershipToAndStop() {
+        Mockito.when(this.rpcService.connect(this.peerId1.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId2.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(this.peerId3.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(this.peerId1);
+        this.replicatorGroup.addReplicator(this.peerId2);
+        this.replicatorGroup.addReplicator(this.peerId3);
+        long logIndex = 8;
+        assertTrue(this.replicatorGroup.transferLeadershipTo(this.peerId1, 8));
+        final Replicator r = (Replicator) this.replicatorGroup.getReplicator(this.peerId1).lock();
+        assertEquals(r.getTimeoutNowIndex(), logIndex);
+        this.replicatorGroup.getReplicator(this.peerId1).unlock();
+        assertTrue(this.replicatorGroup.stopTransferLeadership(this.peerId1));
+        assertEquals(r.getTimeoutNowIndex(), 0);
+    }
+
+    @Test
+    public void testFindTheNextCandidateWithPriority1() {
+        final PeerId p1 = new PeerId("localhost", 18881, 0, 60);
+        final PeerId p2 = new PeerId("localhost", 18882, 0, 80);
+        final PeerId p3 = new PeerId("localhost", 18883, 0, 100);
+        Mockito.when(this.rpcService.connect(p1.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(p2.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(p3.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(p1);
+        this.replicatorGroup.addReplicator(p2);
+        this.replicatorGroup.addReplicator(p3);
+        final ConfigurationEntry conf = new ConfigurationEntry();
+        conf.setConf(new Configuration(Arrays.asList(p1, p2, p3)));
+        final PeerId p = this.replicatorGroup.findTheNextCandidate(conf);
+        assertEquals(p3, p);
+    }
+
+    @Test
+    public void testFindTheNextCandidateWithPriority2() {
+        final PeerId p1 = new PeerId("localhost", 18881, 0, 0);
+        final PeerId p2 = new PeerId("localhost", 18882, 0, 0);
+        final PeerId p3 = new PeerId("localhost", 18883, 0, -1);
+        Mockito.when(this.rpcService.connect(p1.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(p2.getEndpoint())).thenReturn(true);
+        Mockito.when(this.rpcService.connect(p3.getEndpoint())).thenReturn(true);
+        this.replicatorGroup.resetTerm(1);
+        this.replicatorGroup.addReplicator(p1);
+        this.replicatorGroup.addReplicator(p2);
+        this.replicatorGroup.addReplicator(p3);
+        final ConfigurationEntry conf = new ConfigurationEntry();
+        conf.setConf(new Configuration(Arrays.asList(p1, p2, p3)));
+        final PeerId p = this.replicatorGroup.findTheNextCandidate(conf);
+        assertEquals(p3, p);
+    }
+
+    @After
+    public void teardown() {
+        this.timerManager.shutdown();
+        this.errorCounter.set(0);
+        this.stoppedCounter.set(0);
+        this.startedCounter.set(0);
+    }
+
+    private int heartbeatTimeout(final int electionTimeout) {
+        return Math.max(electionTimeout / this.raftOptions.getElectionHeartbeatFactor(), 10);
+    }
+
+    private void mockSendEmptyEntries() {
+        final RpcRequests.AppendEntriesRequest request1 = createEmptyEntriesRequestToPeer(this.peerId1);
+        final RpcRequests.AppendEntriesRequest request2 = createEmptyEntriesRequestToPeer(this.peerId2);
+        final RpcRequests.AppendEntriesRequest request3 = createEmptyEntriesRequestToPeer(this.peerId3);
+
+        Mockito
+            .when(this.rpcService.appendEntries(eq(this.peerId1.getEndpoint()), eq(request1), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+        Mockito
+            .when(this.rpcService.appendEntries(eq(this.peerId2.getEndpoint()), eq(request2), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+        Mockito
+            .when(this.rpcService.appendEntries(eq(this.peerId3.getEndpoint()), eq(request3), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+    }
+
+    private RpcRequests.AppendEntriesRequest createEmptyEntriesRequestToPeer(final PeerId peerId) {
+        return RpcRequests.AppendEntriesRequest.newBuilder() //
+            .setGroupId("test") //
+            .setServerId(new PeerId("localhost", 8081).toString()) //
+            .setPeerId(peerId.toString()) //
+            .setTerm(1) //
+            .setPrevLogIndex(10) //
+            .setPrevLogTerm(1) //
+            .setCommittedIndex(0) //
+            .setData(ByteString.EMPTY) //
+            .build();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java
new file mode 100644
index 0000000..69be89b
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/ReplicatorTest.java
@@ -0,0 +1,809 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.nio.ByteBuffer;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledFuture;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Matchers;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.CatchUpClosure;
+import com.alipay.sofa.jraft.core.Replicator.RequestType;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.ReplicatorOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosureAdapter;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.SnapshotStorage;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.util.ThreadId;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class ReplicatorTest {
+
+    private ThreadId          id;
+    private final RaftOptions raftOptions = new RaftOptions();
+    private TimerManager      timerManager;
+    @Mock
+    private RaftClientService rpcService;
+    @Mock
+    private NodeImpl          node;
+    @Mock
+    private BallotBox         ballotBox;
+    @Mock
+    private LogManager        logManager;
+    @Mock
+    private SnapshotStorage   snapshotStorage;
+    private ReplicatorOptions opts;
+    private final PeerId      peerId      = new PeerId("localhost", 8081);
+
+    @Before
+    public void setup() {
+        this.timerManager = new TimerManager(5);
+        this.opts = new ReplicatorOptions();
+        this.opts.setRaftRpcService(this.rpcService);
+        this.opts.setPeerId(this.peerId);
+        this.opts.setBallotBox(this.ballotBox);
+        this.opts.setGroupId("test");
+        this.opts.setTerm(1);
+        this.opts.setServerId(new PeerId("localhost", 8082));
+        this.opts.setNode(this.node);
+        this.opts.setSnapshotStorage(this.snapshotStorage);
+        this.opts.setTimerManager(this.timerManager);
+        this.opts.setLogManager(this.logManager);
+        this.opts.setDynamicHeartBeatTimeoutMs(100);
+        this.opts.setElectionTimeoutMs(1000);
+
+        Mockito.when(this.logManager.getLastLogIndex()).thenReturn(10L);
+        Mockito.when(this.logManager.getTerm(10)).thenReturn(1L);
+        Mockito.when(this.rpcService.connect(this.peerId.getEndpoint())).thenReturn(true);
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(new NodeMetrics(true));
+        // mock send empty entries
+        mockSendEmptyEntries();
+
+        this.id = Replicator.start(this.opts, this.raftOptions);
+    }
+
+    private void mockSendEmptyEntries() {
+        this.mockSendEmptyEntries(false);
+    }
+
+    private void mockSendEmptyEntries(final boolean isHeartbeat) {
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(isHeartbeat);
+        Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+    }
+
+    private RpcRequests.AppendEntriesRequest createEmptyEntriesRequest() {
+        return this.createEmptyEntriesRequest(false);
+    }
+
+    private RpcRequests.AppendEntriesRequest createEmptyEntriesRequest(final boolean isHeartbeat) {
+        RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+            .setGroupId("test") //
+            .setServerId(new PeerId("localhost", 8082).toString()) //
+            .setPeerId(this.peerId.toString()) //
+            .setTerm(1) //
+            .setPrevLogIndex(10) //
+            .setPrevLogTerm(1) //
+            .setCommittedIndex(0);
+        if (!isHeartbeat) {
+            rb.setData(ByteString.EMPTY);
+        }
+        return rb.build();
+    }
+
+    @After
+    public void teardown() {
+        this.timerManager.shutdown();
+    }
+
+    @Test
+    public void testStartDestroyJoin() throws Exception {
+        assertNotNull(this.id);
+        final Replicator r = getReplicator();
+        assertNotNull(r);
+        assertNotNull(r.getRpcInFly());
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+        assertSame(r.getOpts(), this.opts);
+        this.id.unlock();
+        assertEquals(0, Replicator.getNextIndex(this.id));
+        assertNotNull(r.getHeartbeatTimer());
+        r.destroy();
+        Replicator.join(this.id);
+        assertNull(r.id);
+    }
+
+    @Test
+    public void testMetricRemoveOnDestroy() {
+        assertNotNull(this.id);
+        final Replicator r = getReplicator();
+        assertNotNull(r);
+        assertSame(r.getOpts(), this.opts);
+        Set metrics = this.opts.getNode().getNodeMetrics().getMetricRegistry().getNames();
+        assertEquals(7, metrics.size());
+        r.destroy();
+        metrics = this.opts.getNode().getNodeMetrics().getMetricRegistry().getNames();
+        assertEquals(0, metrics.size());
+    }
+
+    private Replicator getReplicator() {
+        return (Replicator) this.id.lock();
+    }
+
+    @Test
+    public void testOnRpcReturnedRpcError() {
+        testRpcReturnedError();
+    }
+
+    private Replicator testRpcReturnedError() {
+        final Replicator r = getReplicator();
+        assertNull(r.getBlockTimer());
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(12) //
+            .setTerm(2) //
+            .build();
+        this.id.unlock();
+
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+            response, 0, 0, Utils.monotonicMs());
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+        assertNotNull(r.getBlockTimer());
+        return r;
+    }
+
+    @Test
+    public void testOnRpcReturnedRpcContinuousError() throws Exception {
+        Replicator r = testRpcReturnedError();
+        ScheduledFuture timer = r.getBlockTimer();
+        assertNotNull(timer);
+
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(12) //
+            .setTerm(2) //
+            .build();
+        r.getInflights().add(new Replicator.Inflight(RequestType.AppendEntries, r.getNextSendIndex(), 0, 0, 1, null));
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+            response, 1, 1, Utils.monotonicMs());
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+        assertNotNull(r.getBlockTimer());
+        // the same timer
+        assertSame(timer, r.getBlockTimer());
+
+        Thread.sleep(r.getOpts().getDynamicHeartBeatTimeoutMs() * 2);
+        r.getInflights().add(new Replicator.Inflight(RequestType.AppendEntries, r.getNextSendIndex(), 0, 0, 1, null));
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, new Status(-1, "test error"), request,
+            response, 1, 2, Utils.monotonicMs());
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.BLOCKING);
+        assertNotNull(r.getBlockTimer());
+        // the same timer
+        assertNotSame(timer, r.getBlockTimer());
+    }
+
+    @Test
+    public void testOnRpcReturnedTermMismatch() {
+        final Replicator r = getReplicator();
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(12) //
+            .setTerm(2) //
+            .build();
+        this.id.unlock();
+
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+            Utils.monotonicMs());
+        Mockito.verify(this.node).increaseTermTo(
+            2,
+            new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
+                this.peerId));
+        assertNull(r.id);
+    }
+
+    @Test
+    public void testOnRpcReturnedMoreLogs() {
+        final Replicator r = getReplicator();
+        assertEquals(11, r.getRealNextIndex());
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(12) //
+            .setTerm(1) //
+            .build();
+        this.id.unlock();
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+
+        Mockito.when(this.logManager.getTerm(9)).thenReturn(1L);
+        final RpcRequests.AppendEntriesRequest newReq = RpcRequests.AppendEntriesRequest.newBuilder(). //
+            setGroupId("test"). //
+            setServerId(new PeerId("localhost", 8082).toString()). //
+            setPeerId(this.peerId.toString()). //
+            setTerm(1). //
+            setPrevLogIndex(9). //
+            setData(ByteString.EMPTY). //
+            setPrevLogTerm(1). //
+            setCommittedIndex(0).build();
+        Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(newReq), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+            Utils.monotonicMs());
+
+        assertNotNull(r.getRpcInFly());
+        assertNotSame(r.getRpcInFly(), rpcInFly);
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+        this.id.unlock();
+        assertEquals(0, Replicator.getNextIndex(this.id));
+        assertEquals(10, r.getRealNextIndex());
+    }
+
+    @Test
+    public void testOnRpcReturnedLessLogs() {
+        final Replicator r = getReplicator();
+        assertEquals(11, r.getRealNextIndex());
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(8) //
+            .setTerm(1) //
+            .build();
+        this.id.unlock();
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+
+        Mockito.when(this.logManager.getTerm(8)).thenReturn(1L);
+        final RpcRequests.AppendEntriesRequest newReq = RpcRequests.AppendEntriesRequest.newBuilder() //
+            .setGroupId("test") //
+            .setServerId(new PeerId("localhost", 8082).toString()) //
+            .setPeerId(this.peerId.toString()) //
+            .setTerm(1) //
+            .setPrevLogIndex(8) //
+            .setPrevLogTerm(1) //
+            .setData(ByteString.EMPTY) //
+            .setCommittedIndex(0) //
+            .build();
+        Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(newReq), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+            Utils.monotonicMs());
+
+        assertNotNull(r.getRpcInFly());
+        assertNotSame(r.getRpcInFly(), rpcInFly);
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.APPENDING_ENTRIES);
+        this.id.unlock();
+        assertEquals(0, Replicator.getNextIndex(this.id));
+        assertEquals(9, r.getRealNextIndex());
+    }
+
+    @Test
+    public void testOnRpcReturnedWaitMoreEntries() throws Exception {
+        final Replicator r = getReplicator();
+        assertEquals(-1, r.getWaitId());
+
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(true) //
+            .setLastLogIndex(10) //
+            .setTerm(1) //
+            .build();
+        this.id.unlock();
+        Mockito.when(this.logManager.wait(eq(10L), Mockito.any(), same(this.id))).thenReturn(99L);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        Replicator.waitForCaughtUp(this.id, 1, System.currentTimeMillis() + 5000, new CatchUpClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+            Utils.monotonicMs());
+
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.IDLE);
+        this.id.unlock();
+        assertEquals(11, Replicator.getNextIndex(this.id));
+        assertEquals(99, r.getWaitId());
+        latch.await(); //make sure catch up closure is invoked.
+    }
+
+    @Test
+    public void testStop() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNotNull(r.getHeartbeatTimer());
+        assertNotNull(r.getRpcInFly());
+        Replicator.stop(this.id);
+        assertNull(r.id);
+        assertNull(r.getHeartbeatTimer());
+        assertNull(r.getRpcInFly());
+    }
+
+    @Test
+    public void testSetErrorStop() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNotNull(r.getHeartbeatTimer());
+        assertNotNull(r.getRpcInFly());
+        this.id.setError(RaftError.ESTOP.getNumber());
+        this.id.unlock();
+        assertNull(r.id);
+        assertNull(r.getHeartbeatTimer());
+        assertNull(r.getRpcInFly());
+    }
+
+    @Test
+    public void testContinueSendingTimeout() throws Exception {
+        testOnRpcReturnedWaitMoreEntries();
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        mockSendEmptyEntries();
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+        assertTrue(Replicator.continueSending(this.id, RaftError.ETIMEDOUT.getNumber()));
+        assertNotNull(r.getRpcInFly());
+        assertNotSame(rpcInFly, r.getRpcInFly());
+    }
+
+    @Test
+    public void testContinueSendingEntries() throws Exception {
+        testOnRpcReturnedWaitMoreEntries();
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        mockSendEmptyEntries();
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+
+        final RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+            .setGroupId("test") //
+            .setServerId(new PeerId("localhost", 8082).toString()) //
+            .setPeerId(this.peerId.toString()) //
+            .setTerm(1) //
+            .setPrevLogIndex(10) //
+            .setPrevLogTerm(1) //
+            .setCommittedIndex(0);
+
+        int totalDataLen = 0;
+        for (int i = 0; i < 10; i++) {
+            totalDataLen += i;
+            final LogEntry value = new LogEntry();
+            value.setData(ByteBuffer.allocate(i));
+            value.setType(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+            value.setId(new LogId(11 + i, 1));
+            Mockito.when(this.logManager.getEntry(11 + i)).thenReturn(value);
+            rb.addEntries(RaftOutter.EntryMeta.newBuilder().setTerm(1).setType(EnumOutter.EntryType.ENTRY_TYPE_DATA)
+                .setDataLen(i));
+        }
+        rb.setData(ByteString.copyFrom(new byte[totalDataLen]));
+
+        final RpcRequests.AppendEntriesRequest request = rb.build();
+        Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+
+        assertEquals(11, r.statInfo.firstLogIndex);
+        assertEquals(10, r.statInfo.lastLogIndex);
+        Mockito.when(this.logManager.getTerm(20)).thenReturn(1L);
+        assertTrue(Replicator.continueSending(this.id, 0));
+        assertNotNull(r.getRpcInFly());
+        assertNotSame(rpcInFly, r.getRpcInFly());
+        assertEquals(11, r.statInfo.firstLogIndex);
+        assertEquals(20, r.statInfo.lastLogIndex);
+        assertEquals(0, r.getWaitId());
+        assertEquals(r.statInfo.runningState, Replicator.RunningState.IDLE);
+    }
+
+    @Test
+    public void testSetErrorTimeout() throws Exception {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNull(r.getHeartbeatInFly());
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(true);
+        Mockito.when(
+            this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request),
+                eq(this.opts.getElectionTimeoutMs() / 2), Mockito.any())).thenReturn(new FutureImpl<>());
+        this.id.setError(RaftError.ETIMEDOUT.getNumber());
+        Thread.sleep(this.opts.getElectionTimeoutMs() + 1000);
+        assertNotNull(r.getHeartbeatInFly());
+    }
+
+    @Test
+    public void testOnHeartbeatReturnedRpcError() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        final ScheduledFuture timer = r.getHeartbeatTimer();
+        assertNotNull(timer);
+        Replicator.onHeartbeatReturned(this.id, new Status(-1, "test"), createEmptyEntriesRequest(), null,
+            Utils.monotonicMs());
+        assertNotNull(r.getHeartbeatTimer());
+        assertNotSame(timer, r.getHeartbeatTimer());
+    }
+
+    @Test
+    public void testOnHeartbeatReturnedOK() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        final ScheduledFuture timer = r.getHeartbeatTimer();
+        assertNotNull(timer);
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder(). //
+            setSuccess(false). //
+            setLastLogIndex(10).setTerm(1).build();
+        Replicator
+            .onHeartbeatReturned(this.id, Status.OK(), createEmptyEntriesRequest(), response, Utils.monotonicMs());
+        assertNotNull(r.getHeartbeatTimer());
+        assertNotSame(timer, r.getHeartbeatTimer());
+    }
+
+    @Test
+    public void testOnHeartbeatReturnedTermMismatch() {
+        final Replicator r = getReplicator();
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder() //
+            .setSuccess(false) //
+            .setLastLogIndex(12) //
+            .setTerm(2) //
+            .build();
+        this.id.unlock();
+
+        Replicator.onHeartbeatReturned(this.id, Status.OK(), request, response, Utils.monotonicMs());
+        Mockito.verify(this.node).increaseTermTo(
+            2,
+            new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term heartbeat_response from peer:%s",
+                this.peerId));
+        assertNull(r.id);
+    }
+
+    @Test
+    public void testTransferLeadership() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertTrue(Replicator.transferLeadership(this.id, 11));
+        assertEquals(11, r.getTimeoutNowIndex());
+        assertNull(r.getTimeoutNowInFly());
+    }
+
+    @Test
+    public void testStopTransferLeadership() {
+        testTransferLeadership();
+        Replicator.stopTransferLeadership(this.id);
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertNull(r.getTimeoutNowInFly());
+    }
+
+    @Test
+    public void testTransferLeadershipSendTimeoutNow() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        r.setHasSucceeded();
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertNull(r.getTimeoutNowInFly());
+
+        final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+        Mockito.when(
+            this.rpcService.timeoutNow(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(request), eq(-1),
+                Mockito.any())).thenReturn(new FutureImpl<>());
+
+        assertTrue(Replicator.transferLeadership(this.id, 10));
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertNotNull(r.getTimeoutNowInFly());
+    }
+
+    @Test
+    public void testSendHeartbeat() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+
+        assertNull(r.getHeartbeatInFly());
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest(true);
+        Mockito.when(
+            this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request),
+                eq(this.opts.getElectionTimeoutMs() / 2), Mockito.any())).thenReturn(new FutureImpl<>());
+        Replicator.sendHeartbeat(this.id, new RpcResponseClosureAdapter() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+
+            }
+        });
+
+        assertNotNull(r.getHeartbeatInFly());
+
+        assertSame(r, this.id.lock());
+        this.id.unlock();
+    }
+
+    @Test
+    public void testSendTimeoutNowAndStop() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        r.setHasSucceeded();
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertNull(r.getTimeoutNowInFly());
+        assertTrue(Replicator.sendTimeoutNowAndStop(this.id, 10));
+        assertEquals(0, r.getTimeoutNowIndex());
+        assertNull(r.getTimeoutNowInFly());
+        final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+        Mockito.verify(this.rpcService).timeoutNow(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(request),
+            eq(10), Mockito.any());
+    }
+
+    private RpcRequests.TimeoutNowRequest createTimeoutnowRequest() {
+        final RpcRequests.TimeoutNowRequest.Builder rb = RpcRequests.TimeoutNowRequest.newBuilder();
+        rb.setTerm(this.opts.getTerm());
+        rb.setGroupId(this.opts.getGroupId());
+        rb.setServerId(this.opts.getServerId().toString());
+        rb.setPeerId(this.opts.getPeerId().toString());
+        return rb.build();
+    }
+
+    @Test
+    public void testOnTimeoutNowReturnedRpcErrorAndStop() {
+        final Replicator r = getReplicator();
+        final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+        this.id.unlock();
+
+        Replicator.onTimeoutNowReturned(this.id, new Status(-1, "test"), request, null, true);
+        assertNull(r.id);
+    }
+
+    @Test
+    public void testInstallSnapshotNoReader() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+        r.installSnapshot();
+        final ArgumentCaptor errArg = ArgumentCaptor.forClass(RaftException.class);
+        Mockito.verify(this.node).onError(errArg.capture());
+        Assert.assertEquals(RaftError.EIO, errArg.getValue().getStatus().getRaftError());
+        Assert.assertEquals("Fail to open snapshot", errArg.getValue().getStatus().getErrorMsg());
+    }
+
+    @Test
+    public void testInstallSnapshot() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+
+        final Future rpcInFly = r.getRpcInFly();
+        assertNotNull(rpcInFly);
+        final SnapshotReader reader = Mockito.mock(SnapshotReader.class);
+        Mockito.when(this.snapshotStorage.open()).thenReturn(reader);
+        final String uri = "remote://localhost:8081/99";
+        Mockito.when(reader.generateURIForCopy()).thenReturn(uri);
+        final RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder() //
+            .setLastIncludedIndex(11) //
+            .setLastIncludedTerm(1) //
+            .build();
+        Mockito.when(reader.load()).thenReturn(meta);
+
+        assertEquals(0, r.statInfo.lastLogIncluded);
+        assertEquals(0, r.statInfo.lastTermIncluded);
+
+        final RpcRequests.InstallSnapshotRequest.Builder rb = RpcRequests.InstallSnapshotRequest.newBuilder();
+        rb.setTerm(this.opts.getTerm());
+        rb.setGroupId(this.opts.getGroupId());
+        rb.setServerId(this.opts.getServerId().toString());
+        rb.setPeerId(this.opts.getPeerId().toString());
+        rb.setMeta(meta);
+        rb.setUri(uri);
+
+        Mockito.when(
+            this.rpcService.installSnapshot(Matchers.eq(this.opts.getPeerId().getEndpoint()), eq(rb.build()),
+                Mockito.any())).thenReturn(new FutureImpl<>());
+
+        r.installSnapshot();
+        assertNotNull(r.getRpcInFly());
+        assertNotSame(r.getRpcInFly(), rpcInFly);
+        Assert.assertEquals(Replicator.RunningState.INSTALLING_SNAPSHOT, r.statInfo.runningState);
+        assertEquals(11, r.statInfo.lastLogIncluded);
+        assertEquals(1, r.statInfo.lastTermIncluded);
+    }
+
+    @Test
+    public void testOnTimeoutNowReturnedTermMismatch() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        final RpcRequests.TimeoutNowRequest request = createTimeoutnowRequest();
+        final RpcRequests.TimeoutNowResponse response = RpcRequests.TimeoutNowResponse.newBuilder() //
+            .setSuccess(false) //
+            .setTerm(12) //
+            .build();
+        this.id.unlock();
+
+        Replicator.onTimeoutNowReturned(this.id, Status.OK(), request, response, false);
+        Mockito.verify(this.node).increaseTermTo(
+            12,
+            new Status(RaftError.EHIGHERTERMRESPONSE, "Leader receives higher term timeout_now_response from peer:%s",
+                this.peerId));
+        assertNull(r.id);
+    }
+
+    @Test
+    public void testOnInstallSnapshotReturned() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNull(r.getBlockTimer());
+
+        final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+        final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+            .setSuccess(true).setTerm(1).build();
+        assertEquals(-1, r.getWaitId());
+        Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, Status.OK(), request, response, 0, 0, -1);
+        assertNull(r.getBlockTimer());
+        assertEquals(0, r.getWaitId());
+    }
+
+    @Test
+    public void testOnInstallSnapshotReturnedRpcError() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNull(r.getBlockTimer());
+
+        final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+        final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+            .setSuccess(true).setTerm(1).build();
+        assertEquals(-1, r.getWaitId());
+        Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, new Status(-1, "test"), request, response,
+            0, 0, -1);
+        assertNotNull(r.getBlockTimer());
+        assertEquals(-1, r.getWaitId());
+    }
+
+    @Test
+    public void testOnInstallSnapshotReturnedFailure() {
+        final Replicator r = getReplicator();
+        this.id.unlock();
+        assertNull(r.getBlockTimer());
+
+        final RpcRequests.InstallSnapshotRequest request = createInstallSnapshotRequest();
+        final RpcRequests.InstallSnapshotResponse response = RpcRequests.InstallSnapshotResponse.newBuilder()
+            .setSuccess(false).setTerm(1).build();
+        assertEquals(-1, r.getWaitId());
+        Mockito.when(this.logManager.getTerm(11)).thenReturn(1L);
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.Snapshot, Status.OK(), request, response, 0, 0, -1);
+        assertNotNull(r.getBlockTimer());
+        assertEquals(-1, r.getWaitId());
+    }
+
+    @Test
+    public void testOnRpcReturnedOutOfOrder() {
+        final Replicator r = getReplicator();
+        assertEquals(-1, r.getWaitId());
+
+        final RpcRequests.AppendEntriesRequest request = createEmptyEntriesRequest();
+        final RpcRequests.AppendEntriesResponse response = RpcRequests.AppendEntriesResponse.newBuilder(). //
+            setSuccess(true). //
+            setLastLogIndex(10).setTerm(1).build();
+        assertNull(r.getBlockTimer());
+        this.id.unlock();
+
+        assertTrue(r.getPendingResponses().isEmpty());
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 1, 0,
+            Utils.monotonicMs());
+        assertEquals(1, r.getPendingResponses().size());
+        Replicator.onRpcReturned(this.id, Replicator.RequestType.AppendEntries, Status.OK(), request, response, 0, 0,
+            Utils.monotonicMs());
+        assertTrue(r.getPendingResponses().isEmpty());
+        assertEquals(0, r.getWaitId());
+        assertEquals(11, r.getRealNextIndex());
+        assertEquals(1, r.getRequiredNextSeq());
+    }
+
+    private void mockSendEntries(@SuppressWarnings("SameParameterValue") final int n) {
+        final RpcRequests.AppendEntriesRequest request = createEntriesRequest(n);
+        Mockito.when(this.rpcService.appendEntries(eq(this.peerId.getEndpoint()), eq(request), eq(-1), Mockito.any()))
+            .thenReturn(new FutureImpl<>());
+    }
+
+    private RpcRequests.AppendEntriesRequest createEntriesRequest(final int n) {
+        final RpcRequests.AppendEntriesRequest.Builder rb = RpcRequests.AppendEntriesRequest.newBuilder() //
+            .setGroupId("test") //
+            .setServerId(new PeerId("localhost", 8082).toString()) //
+            .setPeerId(this.peerId.toString()) //
+            .setTerm(1) //
+            .setPrevLogIndex(10) //
+            .setPrevLogTerm(1) //
+            .setCommittedIndex(0);
+
+        for (int i = 0; i < n; i++) {
+            final LogEntry log = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_DATA);
+            log.setData(ByteBuffer.wrap(new byte[i]));
+            log.setId(new LogId(i + 11, 1));
+            Mockito.when(this.logManager.getEntry(i + 11)).thenReturn(log);
+            Mockito.when(this.logManager.getTerm(i + 11)).thenReturn(1L);
+            rb.addEntries(RaftOutter.EntryMeta.newBuilder().setDataLen(i).setTerm(1)
+                .setType(EnumOutter.EntryType.ENTRY_TYPE_DATA).build());
+        }
+
+        return rb.build();
+    }
+
+    @Test
+    public void testGetNextSendIndex() {
+        final Replicator r = getReplicator();
+        assertEquals(-1, r.getNextSendIndex());
+        r.resetInflights();
+        assertEquals(11, r.getNextSendIndex());
+        mockSendEntries(3);
+        r.sendEntries();
+        assertEquals(14, r.getNextSendIndex());
+    }
+
+    private RpcRequests.InstallSnapshotRequest createInstallSnapshotRequest() {
+        final String uri = "remote://localhost:8081/99";
+        final RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder() //
+            .setLastIncludedIndex(11) //
+            .setLastIncludedTerm(1) //
+            .build();
+        final RpcRequests.InstallSnapshotRequest.Builder rb = RpcRequests.InstallSnapshotRequest.newBuilder();
+        rb.setTerm(this.opts.getTerm());
+        rb.setGroupId(this.opts.getGroupId());
+        rb.setServerId(this.opts.getServerId().toString());
+        rb.setPeerId(this.opts.getPeerId().toString());
+        rb.setMeta(meta);
+        rb.setUri(uri);
+        return rb.build();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
new file mode 100644
index 0000000..aa1bf5e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
@@ -0,0 +1,494 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.JRaftServiceFactory;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Test cluster for NodeTest
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-20 1:41:17 PM
+ */
+public class TestCluster {
+
+    static class Clusters {
+
+        public final IdentityHashMap needCloses = new IdentityHashMap<>();
+        private final Object                              EXIST      = new Object();
+
+        public synchronized void add(final TestCluster cluster) {
+            this.needCloses.put(cluster, EXIST);
+        }
+
+        public synchronized boolean remove(final TestCluster cluster) {
+            return this.needCloses.remove(cluster) != null;
+        }
+
+        public synchronized boolean isEmpty() {
+            return this.needCloses.isEmpty();
+        }
+
+        public synchronized List removeAll() {
+            final List clusters = new ArrayList<>(this.needCloses.keySet());
+            this.needCloses.clear();
+            return clusters;
+        }
+    }
+
+    public static final Clusters                          CLUSTERS           = new Clusters();
+
+    private final String                                  dataPath;
+    private final String                                  name;                                              // groupId
+    private final List                            peers;
+    private final List                          nodes;
+    private final LinkedHashMap fsms;
+    private final ConcurrentMap serverMap          = new ConcurrentHashMap<>();
+    private final int                                     electionTimeoutMs;
+    private final Lock                                    lock               = new ReentrantLock();
+
+    private JRaftServiceFactory                           raftServiceFactory = new TestJRaftServiceFactory();
+
+    private LinkedHashSet                         learners;
+
+    public JRaftServiceFactory getRaftServiceFactory() {
+        return this.raftServiceFactory;
+    }
+
+    public void setRaftServiceFactory(final JRaftServiceFactory raftServiceFactory) {
+        this.raftServiceFactory = raftServiceFactory;
+    }
+
+    public LinkedHashSet getLearners() {
+        return this.learners;
+    }
+
+    public void setLearners(final LinkedHashSet learners) {
+        this.learners = learners;
+    }
+
+    public List getPeers() {
+        return this.peers;
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers) {
+        this(name, dataPath, peers, 300);
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers, final int electionTimeoutMs) {
+        this(name, dataPath, peers, new LinkedHashSet<>(), 300);
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers,
+                       final LinkedHashSet learners, final int electionTimeoutMs) {
+        super();
+        this.name = name;
+        this.dataPath = dataPath;
+        this.peers = peers;
+        this.nodes = new ArrayList<>(this.peers.size());
+        this.fsms = new LinkedHashMap<>(this.peers.size());
+        this.electionTimeoutMs = electionTimeoutMs;
+        this.learners = learners;
+        CLUSTERS.add(this);
+    }
+
+    public boolean start(final Endpoint addr) throws Exception {
+        return this.start(addr, false, 300);
+    }
+
+    public boolean start(final Endpoint addr, final int priority) throws Exception {
+        return this.start(addr, false, 300, false, null, null, priority);
+    }
+
+    public boolean startLearner(final PeerId peer) throws Exception {
+        this.learners.add(peer);
+        return this.start(peer.getEndpoint(), false, 300);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs)
+                                                                                                             throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, false);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics) throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, null, null);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle) throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, snapshotThrottle, null);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+                         final RaftOptions raftOptions, final int priority) throws IOException {
+
+        if (this.serverMap.get(listenAddr.toString()) != null) {
+            return true;
+        }
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+        nodeOptions.setEnableMetrics(enableMetrics);
+        nodeOptions.setSnapshotThrottle(snapshotThrottle);
+        nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+        nodeOptions.setServiceFactory(this.raftServiceFactory);
+        if (raftOptions != null) {
+            nodeOptions.setRaftOptions(raftOptions);
+        }
+        final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        FileUtils.forceMkdir(new File(serverDataPath));
+        nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+        nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+        nodeOptions.setElectionPriority(priority);
+
+        final MockStateMachine fsm = new MockStateMachine(listenAddr);
+        nodeOptions.setFsm(fsm);
+
+        if (!emptyPeers) {
+            nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+        }
+
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+        final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0, priority),
+            nodeOptions, rpcServer);
+
+        this.lock.lock();
+        try {
+            if (this.serverMap.put(listenAddr.toString(), server) == null) {
+                final Node node = server.start();
+
+                this.fsms.put(new PeerId(listenAddr, 0), fsm);
+                this.nodes.add((NodeImpl) node);
+                return true;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return false;
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+                         final RaftOptions raftOptions) throws IOException {
+
+        if (this.serverMap.get(listenAddr.toString()) != null) {
+            return true;
+        }
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+        nodeOptions.setEnableMetrics(enableMetrics);
+        nodeOptions.setSnapshotThrottle(snapshotThrottle);
+        nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+        nodeOptions.setServiceFactory(this.raftServiceFactory);
+        if (raftOptions != null) {
+            nodeOptions.setRaftOptions(raftOptions);
+        }
+        final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        FileUtils.forceMkdir(new File(serverDataPath));
+        nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+        nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+        final MockStateMachine fsm = new MockStateMachine(listenAddr);
+        nodeOptions.setFsm(fsm);
+
+        if (!emptyPeers) {
+            nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+        }
+
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+        final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0), nodeOptions,
+            rpcServer);
+
+        this.lock.lock();
+        try {
+            if (this.serverMap.put(listenAddr.toString(), server) == null) {
+                final Node node = server.start();
+
+                this.fsms.put(new PeerId(listenAddr, 0), fsm);
+                this.nodes.add((NodeImpl) node);
+                return true;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return false;
+    }
+
+    public MockStateMachine getFsmByPeer(final PeerId peer) {
+        this.lock.lock();
+        try {
+            return this.fsms.get(peer);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public List getFsms() {
+        this.lock.lock();
+        try {
+            return new ArrayList<>(this.fsms.values());
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public boolean stop(final Endpoint listenAddr) throws InterruptedException {
+        final Node node = removeNode(listenAddr);
+        final CountDownLatch latch = new CountDownLatch(1);
+        if (node != null) {
+            node.shutdown(new ExpectClosure(latch));
+            node.join();
+            latch.await();
+        }
+        final RaftGroupService raftGroupService = this.serverMap.remove(listenAddr.toString());
+        raftGroupService.shutdown();
+        raftGroupService.join();
+        return node != null;
+    }
+
+    public void stopAll() throws InterruptedException {
+        final List addrs = getAllNodes();
+        final List nodes = new ArrayList<>();
+        for (final Endpoint addr : addrs) {
+            final Node node = removeNode(addr);
+            node.shutdown();
+            nodes.add(node);
+            this.serverMap.remove(addr.toString()).shutdown();
+        }
+        for (final Node node : nodes) {
+            node.join();
+        }
+        CLUSTERS.remove(this);
+    }
+
+    public void clean(final Endpoint listenAddr) throws IOException {
+        final String path = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        System.out.println("Clean dir:" + path);
+        FileUtils.deleteDirectory(new File(path));
+    }
+
+    public Node getLeader() {
+        this.lock.lock();
+        try {
+            for (int i = 0; i < this.nodes.size(); i++) {
+                final NodeImpl node = this.nodes.get(i);
+                if (node.isLeader() && this.fsms.get(node.getServerId()).getLeaderTerm() == node.getCurrentTerm()) {
+                    return node;
+                }
+            }
+            return null;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public MockStateMachine getLeaderFsm() {
+        final Node leader = getLeader();
+        if (leader != null) {
+            return (MockStateMachine) leader.getOptions().getFsm();
+        }
+        return null;
+    }
+
+    public void waitLeader() throws InterruptedException {
+        while (true) {
+            final Node node = getLeader();
+            if (node != null) {
+                return;
+            } else {
+                Thread.sleep(10);
+            }
+        }
+    }
+
+    public List getFollowers() {
+        final List ret = new ArrayList<>();
+        this.lock.lock();
+        try {
+            for (final NodeImpl node : this.nodes) {
+                if (!node.isLeader() && !this.learners.contains(node.getServerId())) {
+                    ret.add(node);
+                }
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return ret;
+    }
+
+    /**
+     * Ensure all peers leader is expectAddr
+     * @param expectAddr expected address
+     * @throws InterruptedException if interrupted
+     */
+    public void ensureLeader(final Endpoint expectAddr) throws InterruptedException {
+        while (true) {
+            this.lock.lock();
+            for (final Node node : this.nodes) {
+                final PeerId leaderId = node.getLeaderId();
+                if (!leaderId.getEndpoint().equals(expectAddr)) {
+                    this.lock.unlock();
+                    Thread.sleep(10);
+                    continue;
+                }
+            }
+            // all is ready
+            this.lock.unlock();
+            return;
+        }
+    }
+
+    public List getNodes() {
+        this.lock.lock();
+        try {
+            return new ArrayList<>(this.nodes);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public List getAllNodes() {
+        this.lock.lock();
+        try {
+            return this.nodes.stream().map(node -> node.getNodeId().getPeerId().getEndpoint())
+                .collect(Collectors.toList());
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public Node removeNode(final Endpoint addr) {
+        Node ret = null;
+        this.lock.lock();
+        try {
+            for (int i = 0; i < this.nodes.size(); i++) {
+                if (this.nodes.get(i).getNodeId().getPeerId().getEndpoint().equals(addr)) {
+                    ret = this.nodes.remove(i);
+                    this.fsms.remove(ret.getNodeId().getPeerId());
+                    break;
+                }
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return ret;
+    }
+
+    public boolean ensureSame() throws InterruptedException {
+        return this.ensureSame(-1);
+    }
+
+    /**
+     * Ensure all logs is the same in all nodes.
+     * @param waitTimes
+     * @return
+     * @throws InterruptedException
+     */
+    public boolean ensureSame(final int waitTimes) throws InterruptedException {
+        this.lock.lock();
+        List fsmList = new ArrayList<>(this.fsms.values());
+        if (fsmList.size() <= 1) {
+            this.lock.unlock();
+            return true;
+        }
+        System.out.println("Start ensureSame, waitTimes=" + waitTimes);
+        try {
+            int nround = 0;
+            final MockStateMachine first = fsmList.get(0);
+            CHECK: while (true) {
+                first.lock();
+                if (first.getLogs().isEmpty()) {
+                    first.unlock();
+                    Thread.sleep(10);
+                    nround++;
+                    if (waitTimes > 0 && nround > waitTimes) {
+                        return false;
+                    }
+                    continue CHECK;
+                }
+
+                for (int i = 1; i < fsmList.size(); i++) {
+                    final MockStateMachine fsm = fsmList.get(i);
+                    fsm.lock();
+                    if (fsm.getLogs().size() != first.getLogs().size()) {
+                        fsm.unlock();
+                        first.unlock();
+                        Thread.sleep(10);
+                        nround++;
+                        if (waitTimes > 0 && nround > waitTimes) {
+                            return false;
+                        }
+                        continue CHECK;
+                    }
+
+                    for (int j = 0; j < first.getLogs().size(); j++) {
+                        final ByteBuffer firstData = first.getLogs().get(j);
+                        final ByteBuffer fsmData = fsm.getLogs().get(j);
+                        if (!firstData.equals(fsmData)) {
+                            fsm.unlock();
+                            first.unlock();
+                            Thread.sleep(10);
+                            nround++;
+                            if (waitTimes > 0 && nround > waitTimes) {
+                                return false;
+                            }
+                            continue CHECK;
+                        }
+                    }
+                    fsm.unlock();
+                }
+                first.unlock();
+                break;
+            }
+            return true;
+        } finally {
+            this.lock.unlock();
+            System.out.println("End ensureSame, waitTimes=" + waitTimes);
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java
new file mode 100644
index 0000000..01a9345
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/TestJRaftServiceFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage;
+
+public class TestJRaftServiceFactory extends DefaultJRaftServiceFactory {
+
+    @Override
+    public LogStorage createLogStorage(final String uri, final RaftOptions raftOptions) {
+        return RocksDBSegmentLogStorage.builder(uri, raftOptions) //
+            .setPreAllocateSegmentCount(1) //
+            .setKeepInMemorySegmentCount(2) //
+            .setMaxSegmentFileSize(512 * 1024) //
+            .setValueSizeThreshold(0) //
+            .build();
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java
new file mode 100644
index 0000000..9a9209d
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/core/V1JRaftServiceFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+import com.alipay.sofa.jraft.entity.codec.v1.LogEntryV1CodecFactory;
+
+public class V1JRaftServiceFactory extends DefaultJRaftServiceFactory {
+
+    @Override
+    public LogEntryCodecFactory createLogEntryCodecFactory() {
+        return LogEntryV1CodecFactory.getInstance();
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java
new file mode 100644
index 0000000..f82388b
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/BallotTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class BallotTest {
+
+    private Ballot ballot;
+
+    @Before
+    public void setup() {
+        this.ballot = new Ballot();
+        this.ballot.init(JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083"), null);
+    }
+
+    @Test
+    public void testGrant() {
+        PeerId peer1 = new PeerId("localhost", 8081);
+        this.ballot.grant(peer1);
+        assertFalse(this.ballot.isGranted());
+
+        PeerId unfoundPeer = new PeerId("localhost", 8084);
+        this.ballot.grant(unfoundPeer);
+        assertFalse(this.ballot.isGranted());
+
+        PeerId peer2 = new PeerId("localhost", 8082);
+        this.ballot.grant(peer2);
+        assertTrue(this.ballot.isGranted());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java
new file mode 100644
index 0000000..da5fe73
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogEntryTest.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import java.nio.ByteBuffer;
+import java.nio.ReadOnlyBufferException;
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.codec.v1.LogEntryV1CodecFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class LogEntryTest {
+
+    @SuppressWarnings("deprecation")
+    @Test
+    public void testEncodeDecodeWithoutData() {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        assertSame(LogEntry.EMPTY_DATA, entry.getData());
+        assertNull(entry.getOldPeers());
+
+        byte[] content = entry.encode();
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+        assertEquals(LogEntryV1CodecFactory.MAGIC, content[0]);
+
+        LogEntry nentry = new LogEntry();
+        assertTrue(nentry.decode(content));
+
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+        Assert.assertEquals(EnumOutter.EntryType.ENTRY_TYPE_NO_OP, nentry.getType());
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertSame(LogEntry.EMPTY_DATA, entry.getData());
+        assertNull(nentry.getOldPeers());
+    }
+
+    @SuppressWarnings("deprecation")
+    @Test
+    public void testEncodeDecodeWithData() {
+        ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setData(buf);
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        assertEquals(buf, entry.getData());
+
+        byte[] content = entry.encode();
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+        assertEquals(LogEntryV1CodecFactory.MAGIC, content[0]);
+
+        LogEntry nentry = new LogEntry();
+        assertTrue(nentry.decode(content));
+
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertEquals(buf, nentry.getData());
+        assertEquals(0, nentry.getData().position());
+        assertEquals(5, nentry.getData().remaining());
+        assertNull(nentry.getOldPeers());
+    }
+
+    @Test
+    public void testChecksum() {
+        ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setData(buf);
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+
+        long c = entry.checksum();
+        assertTrue(c != 0);
+        assertEquals(c, entry.checksum());
+        assertFalse(entry.isCorrupted());
+
+        assertFalse(entry.hasChecksum());
+        entry.setChecksum(c);
+        assertTrue(entry.hasChecksum());
+        assertFalse(entry.isCorrupted());
+
+        // modify index, detect corrupted.
+        entry.getId().setIndex(1);
+        assertNotEquals(c, entry.checksum());
+        assertTrue(entry.isCorrupted());
+        // fix index
+        entry.getId().setIndex(100);
+        assertFalse(entry.isCorrupted());
+
+        // modify data, detect corrupted
+        entry.setData(ByteBuffer.wrap("hEllo".getBytes()));
+        assertNotEquals(c, entry.checksum());
+        assertTrue(entry.isCorrupted());
+    }
+
+    @Test
+    public void testSliceReadOnlyData() {
+        ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setData(buf);
+        assertSame(buf, entry.getData());
+        final ByteBuffer slice = entry.sliceData();
+        assertNotSame(buf, slice);
+        assertEquals(5, slice.remaining());
+        assertEquals("hello", new String(slice.array()));
+        slice.position(4);
+        assertEquals(4, slice.position());
+        assertEquals(0, entry.getData().position());
+        slice.put((byte) 'a');
+        assertEquals(97, slice.get(4));
+        assertEquals("hella", new String(entry.getData().array()));
+
+        final ByteBuffer readOnly = entry.getReadOnlyData();
+        assertNotSame(buf, readOnly);
+        assertEquals(5, readOnly.remaining());
+        byte[] bs = new byte[5];
+        readOnly.get(bs);
+        assertEquals("hella", new String(bs));
+
+        try {
+            readOnly.position(4);
+            readOnly.put((byte) 1);
+            fail();
+        } catch (ReadOnlyBufferException e) {
+
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java
new file mode 100644
index 0000000..fcee0a2
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/LogIdTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class LogIdTest {
+
+    @Test
+    public void testCompareTo() {
+        LogId logId = new LogId();
+        assertEquals(0, logId.getIndex());
+        assertEquals(0, logId.getTerm());
+
+        assertTrue(new LogId(1, 0).compareTo(logId) > 0);
+        assertTrue(new LogId(0, 1).compareTo(logId) > 0);
+
+        logId = new LogId(1, 2);
+        assertTrue(new LogId(0, 1).compareTo(logId) < 0);
+        assertTrue(new LogId(0, 2).compareTo(logId) < 0);
+        assertTrue(new LogId(3, 1).compareTo(logId) < 0);
+        assertTrue(new LogId(1, 2).compareTo(logId) == 0);
+    }
+
+    @Test
+    public void testChecksum() {
+        LogId logId = new LogId();
+        logId.setIndex(1);
+        logId.setTerm(2);
+        long c = logId.checksum();
+        assertTrue(c != 0);
+        assertEquals(c, logId.checksum());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java
new file mode 100644
index 0000000..a83910a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/PeerIdTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class PeerIdTest {
+
+    @Test
+    public void testToStringParse() {
+        final PeerId peer = new PeerId("192.168.1.1", 8081, 0);
+        assertEquals("192.168.1.1:8081", peer.toString());
+
+        final PeerId pp = new PeerId();
+        assertTrue(pp.parse(peer.toString()));
+        assertEquals(8081, pp.getPort());
+        assertEquals("192.168.1.1", pp.getIp());
+        assertEquals(0, pp.getIdx());
+        assertEquals(pp, peer);
+        assertEquals(pp.hashCode(), peer.hashCode());
+    }
+
+    @Test
+    public void testIsPriorityNotElected() {
+
+        final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+        final PeerId peer1 = new PeerId(endpoint1, 0, 0);
+        assertEquals("192.168.1.1:8081::0", peer1.toString());
+        assertTrue(peer1.isPriorityNotElected());
+    }
+
+    @Test
+    public void testIsPriorityDisabled() {
+
+        final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+        final PeerId peer1 = new PeerId(endpoint1, 0);
+        assertEquals("192.168.1.1:8081", peer1.toString());
+        assertTrue(peer1.isPriorityDisabled());
+    }
+
+    @Test
+    public void testToStringParseWithIdxAndPriority() {
+
+        // 1.String format is, ip:port::priority
+        final Endpoint endpoint1 = new Endpoint("192.168.1.1", 8081);
+        final PeerId peer1 = new PeerId(endpoint1, 0, 100);
+        assertEquals("192.168.1.1:8081::100", peer1.toString());
+
+        final PeerId p1 = new PeerId();
+        final String str1 = "192.168.1.1:8081::100";
+        assertTrue(p1.parse(str1));
+        assertEquals(8081, p1.getPort());
+        assertEquals("192.168.1.1", p1.getIp());
+        assertEquals(0, p1.getIdx());
+        assertEquals(100, p1.getPriority());
+
+        assertEquals(p1, peer1);
+        assertEquals(p1.hashCode(), peer1.hashCode());
+
+        // 2.String format is, ip:port:idx:priority
+        final Endpoint endpoint2 = new Endpoint("192.168.1.1", 8081);
+        final PeerId peer2 = new PeerId(endpoint2, 100, 200);
+        assertEquals("192.168.1.1:8081:100:200", peer2.toString());
+
+        final PeerId p2 = new PeerId();
+        final String str2 = "192.168.1.1:8081:100:200";
+        assertTrue(p2.parse(str2));
+        assertEquals(8081, p2.getPort());
+        assertEquals("192.168.1.1", p2.getIp());
+        assertEquals(100, p2.getIdx());
+        assertEquals(200, p2.getPriority());
+
+        assertEquals(p2, peer2);
+        assertEquals(p2.hashCode(), peer2.hashCode());
+    }
+
+    @Test
+    public void testIdx() {
+        final PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+        assertEquals("192.168.1.1:8081:1", peer.toString());
+        assertFalse(peer.isEmpty());
+
+        final PeerId pp = new PeerId();
+        assertTrue(pp.parse(peer.toString()));
+        assertEquals(8081, pp.getPort());
+        assertEquals("192.168.1.1", pp.getIp());
+        assertEquals(1, pp.getIdx());
+        assertEquals(pp, peer);
+        assertEquals(pp.hashCode(), peer.hashCode());
+    }
+
+    @Test
+    public void testParseFail() {
+        final PeerId peer = new PeerId();
+        assertTrue(peer.isEmpty());
+        assertFalse(peer.parse("localhsot:2:3:4:5"));
+        assertTrue(peer.isEmpty());
+    }
+
+    @Test
+    public void testEmptyPeer() {
+        PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+        assertFalse(peer.isEmpty());
+        peer = PeerId.emptyPeer();
+        assertTrue(peer.isEmpty());
+    }
+
+    @Test
+    public void testChecksum() {
+        PeerId peer = new PeerId("192.168.1.1", 8081, 1);
+        long c = peer.checksum();
+        assertTrue(c != 0);
+        assertEquals(c, peer.checksum());
+    }
+
+    @Test
+    public void testToStringParseFailed() {
+        final PeerId pp = new PeerId();
+        final String str1 = "";
+        final String str2 = "192.168.1.1";
+        final String str3 = "92.168.1.1:8081::1:2";
+        assertFalse(pp.parse(str1));
+        assertFalse(pp.parse(str2));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java
new file mode 100644
index 0000000..0e57177
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/BaseLogEntryCodecFactoryTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public abstract class BaseLogEntryCodecFactoryTest {
+
+    protected LogEntryEncoder encoder;
+    protected LogEntryDecoder decoder;
+
+    @Before
+    public void setup() {
+        LogEntryCodecFactory factory = newFactory();
+        this.encoder = factory.encoder();
+        this.decoder = factory.decoder();
+    }
+
+    protected abstract LogEntryCodecFactory newFactory();
+
+    @Test
+    public void testEmptyData() {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        entry.setData(ByteBuffer.allocate(0));
+
+        byte[] content = this.encoder.encode(entry);
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+
+        LogEntry nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+        assertNotNull(nentry.getData());
+        assertEquals(0, nentry.getData().remaining());
+    }
+
+    @Test
+    public void testEncodeDecodeEmpty() {
+        try {
+            assertNull(this.encoder.encode(null));
+            fail();
+        } catch (NullPointerException e) {
+            assertTrue(true);
+        }
+        assertNull(this.decoder.decode(null));
+        assertNull(this.decoder.decode(new byte[0]));
+    }
+
+    @Test
+    public void testEncodeDecodeWithoutData() {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        assertSame(LogEntry.EMPTY_DATA, entry.getData());
+        assertNull(entry.getOldPeers());
+
+        byte[] content = this.encoder.encode(entry);
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+
+        LogEntry nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+        Assert.assertEquals(EnumOutter.EntryType.ENTRY_TYPE_NO_OP, nentry.getType());
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertSame(LogEntry.EMPTY_DATA, nentry.getData());
+        assertNull(nentry.getOldPeers());
+    }
+
+    @Test
+    public void testEncodeDecodeWithData() {
+        ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setData(buf);
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        assertEquals(buf, entry.getData());
+
+        byte[] content = this.encoder.encode(entry);
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+
+        LogEntry nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertEquals(buf, nentry.getData());
+        assertEquals(0, nentry.getData().position());
+        assertEquals(5, nentry.getData().remaining());
+        assertNull(nentry.getOldPeers());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java
new file mode 100644
index 0000000..b69c69f
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/LogEntryCodecPerfTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.codec.v1.V1Decoder;
+import com.alipay.sofa.jraft.entity.codec.v1.V1Encoder;
+import com.alipay.sofa.jraft.entity.codec.v2.V2Encoder;
+import com.alipay.sofa.jraft.util.Utils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class LogEntryCodecPerfTest {
+
+    static byte[]            DATA    = new byte[512];
+
+    static {
+        ThreadLocalRandom.current().nextBytes(DATA);
+    }
+
+    static final int         TIMES   = 100000;
+
+    static final int         THREADS = 20;
+
+    private final AtomicLong logSize = new AtomicLong(0);
+
+    @Before
+    public void setup() throws Exception {
+        this.logSize.set(0);
+        System.gc();
+    }
+
+    private void testEncodeDecode(final LogEntryEncoder encoder, final LogEntryDecoder decoder,
+                                  final CyclicBarrier barrier) throws Exception {
+        ByteBuffer buf = ByteBuffer.wrap(DATA);
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setData(buf);
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+
+        if (barrier != null) {
+            barrier.await();
+        }
+
+        for (int i = 0; i < TIMES; i++) {
+            entry.setId(new LogId(i, i));
+            byte[] content = encoder.encode(entry);
+            assert (content.length > 0);
+            this.logSize.addAndGet(content.length);
+            LogEntry nLog = decoder.decode(content);
+            assertEquals(2, nLog.getPeers().size());
+            assertArrayEquals(DATA, nLog.getData().array());
+            assertEquals(i, nLog.getId().getIndex());
+            assertEquals(i, nLog.getId().getTerm());
+        }
+
+        if (barrier != null) {
+            barrier.await();
+        }
+
+    }
+
+    @Test
+    public void testV1Codec() throws Exception {
+        LogEntryEncoder encoder = V1Encoder.INSTANCE;
+        LogEntryDecoder decoder = V1Decoder.INSTANCE;
+        testEncodeDecode(encoder, decoder, null);
+        concurrentTest("V1", encoder, decoder);
+    }
+
+    @Test
+    public void testV2Codec() throws Exception {
+        LogEntryEncoder encoder = V2Encoder.INSTANCE;
+        LogEntryDecoder decoder = AutoDetectDecoder.INSTANCE;
+        testEncodeDecode(encoder, decoder, null);
+        concurrentTest("V2", encoder, decoder);
+    }
+
+    private void concurrentTest(final String version, final LogEntryEncoder encoder, final LogEntryDecoder decoder)
+                                                                                                                   throws InterruptedException,
+                                                                                                                   BrokenBarrierException {
+        final CyclicBarrier barrier = new CyclicBarrier(THREADS + 1);
+        for (int i = 0; i < THREADS; i++) {
+            new Thread(() -> {
+                try {
+                    testEncodeDecode(encoder, decoder, barrier);
+                } catch (Exception e) {
+                    e.printStackTrace(); // NOPMD
+                    fail();
+                }
+            }).start();
+        }
+        long start = Utils.monotonicMs();
+        barrier.await();
+        barrier.await();
+        System.out.println(version + " codec cost:" + (Utils.monotonicMs() - start) + " ms.");
+        System.out.println("Total log size:" + this.logSize.get() + " bytes.");
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java
new file mode 100644
index 0000000..09e985c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v1/LogEntryV1CodecFactoryTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec.v1;
+
+import com.alipay.sofa.jraft.entity.codec.BaseLogEntryCodecFactoryTest;
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+
+public class LogEntryV1CodecFactoryTest extends BaseLogEntryCodecFactoryTest {
+
+    @Override
+    protected LogEntryCodecFactory newFactory() {
+        LogEntryCodecFactory factory = LogEntryV1CodecFactory.getInstance();
+        return factory;
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactoryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactoryTest.java
new file mode 100644
index 0000000..897a3b9
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/entity/codec/v2/LogEntryV2CodecFactoryTest.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.entity.codec.v2;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.codec.BaseLogEntryCodecFactoryTest;
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+import com.alipay.sofa.jraft.entity.codec.v1.V1Encoder;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+public class LogEntryV2CodecFactoryTest extends BaseLogEntryCodecFactoryTest {
+
+    @Override
+    protected LogEntryCodecFactory newFactory() {
+        LogEntryCodecFactory factory = LogEntryV2CodecFactory.getInstance();
+        return factory;
+    }
+
+    @Test
+    public void testEncodeDecodeWithLearners() {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        List theLearners = createLearners("192.168.1.1:8081", "192.168.1.2:8081");
+        entry.setLearners(theLearners);
+        assertSame(entry.getData(), LogEntry.EMPTY_DATA);
+        assertNull(entry.getOldPeers());
+
+        byte[] content = this.encoder.encode(entry);
+
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+
+        LogEntry nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+        assertNull(nentry.getOldLearners());
+        assertPeersAndLearners(theLearners, nentry);
+
+        // test old learners
+        List theOldLearners = createLearners("192.168.1.1:8081");
+        entry.setOldLearners(theOldLearners);
+        content = this.encoder.encode(entry);
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+        nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+        assertPeersAndLearners(theLearners, nentry);
+        List oldLearners = nentry.getOldLearners();
+        assertNotNull(oldLearners);
+        assertEquals(1, oldLearners.size());
+        assertEquals(oldLearners, theOldLearners);
+
+    }
+
+    private void assertPeersAndLearners(final List theLearners, final LogEntry nentry) {
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+        Assert.assertEquals(EnumOutter.EntryType.ENTRY_TYPE_NO_OP, nentry.getType());
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertSame(nentry.getData(), LogEntry.EMPTY_DATA);
+        assertNull(nentry.getOldPeers());
+
+        assertTrue(nentry.hasLearners());
+        List learners = nentry.getLearners();
+        assertNotNull(learners);
+        assertEquals(2, learners.size());
+        assertEquals(learners, theLearners);
+    }
+
+    private List createLearners(final String... peers) {
+        List ret = new ArrayList<>();
+        for (String s : peers) {
+            PeerId e = new PeerId();
+            e.parse(s);
+            ret.add(e);
+        }
+        return ret;
+    }
+
+    @Test
+    public void testDecodeV1LogEntry() {
+
+        ByteBuffer buf = ByteBuffer.wrap("hello".getBytes());
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(100, 3));
+        entry.setData(buf);
+        entry.setPeers(Arrays.asList(new PeerId("localhost", 99, 1), new PeerId("localhost", 100, 2)));
+        assertEquals(buf, entry.getData());
+
+        byte[] content = V1Encoder.INSTANCE.encode(entry);
+        assertNotNull(content);
+        assertTrue(content.length > 0);
+
+        // Decode by auto detect decoder
+        LogEntry nentry = this.decoder.decode(content);
+        assertNotNull(nentry);
+
+        assertEquals(100, nentry.getId().getIndex());
+        assertEquals(3, nentry.getId().getTerm());
+
+        assertEquals(2, nentry.getPeers().size());
+        assertEquals("localhost:99:1", nentry.getPeers().get(0).toString());
+        assertEquals("localhost:100:2", nentry.getPeers().get(1).toString());
+        assertEquals(buf, nentry.getData());
+        assertEquals(0, nentry.getData().position());
+        assertEquals(5, nentry.getData().remaining());
+        assertNull(nentry.getOldPeers());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
new file mode 100644
index 0000000..bdb1409
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.InvokeTimeoutException;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RemotingException;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.rpc.impl.AbstractClientService;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class AbstractClientServiceTest {
+    static class MockClientService extends AbstractClientService {
+        public void setRpcClient(final RpcClient rpcClient) {
+            this.rpcClient = rpcClient;
+        }
+    }
+
+    private RpcOptions         rpcOptions;
+    private MockClientService  clientService;
+    @Mock
+    private RpcClient          rpcClient;
+    private RpcResponseFactory rpcResponseFactory = RpcFactoryHelper.responseFactory();
+    private final Endpoint     endpoint           = new Endpoint("localhost", 8081);
+
+    @Before
+    public void setup() {
+        this.rpcOptions = new RpcOptions();
+        this.clientService = new MockClientService();
+        assertTrue(this.clientService.init(this.rpcOptions));
+        this.clientService.setRpcClient(this.rpcClient);
+
+    }
+
+    @Test
+    public void testConnect() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenReturn(this.rpcResponseFactory.newResponse(null, Status.OK()));
+        assertTrue(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testConnectFailure() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenReturn(this.rpcResponseFactory.newResponse(null, new Status(-1, "test")));
+        assertFalse(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testConnectException() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenThrow(new RemotingException("test"));
+        assertFalse(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testDisconnect() {
+        this.clientService.disconnect(this.endpoint);
+        Mockito.verify(this.rpcClient).closeConnection(this.endpoint);
+    }
+
+    static class MockRpcResponseClosure extends RpcResponseClosureAdapter {
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        Status         status;
+
+        @Override
+        public void run(final Status status) {
+            this.status = status;
+            this.latch.countDown();
+        }
+
+    }
+
+    @Test
+    public void testCancel() throws Exception {
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        future.cancel(true);
+        ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+        cb.complete(response, null);
+
+        // The closure should be notified with ECANCELED error code.
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.ECANCELED.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneOK() throws Exception {
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+        cb.complete(response, null);
+
+        Message msg = future.get();
+        assertNotNull(msg);
+        assertTrue(msg instanceof ErrorResponse);
+        assertSame(msg, response);
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(0, done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneException() throws Exception {
+        InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        Mockito
+            .doThrow(new RemotingException())
+            .when(this.rpcClient)
+            .invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx), callbackArg.capture(),
+                eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertTrue(future.isDone());
+
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof RemotingException);
+        }
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.EINTERNAL.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneOnException() throws Exception {
+        InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        cb.complete(null, new InvokeTimeoutException());
+
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof InvokeTimeoutException);
+        }
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.ETIMEDOUT.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDOneOnErrorResponse() throws Exception {
+        final InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        final ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        final CliRequests.GetPeersRequest request = CliRequests.GetPeersRequest.newBuilder() //
+            .setGroupId("id") //
+            .setLeaderId("127.0.0.1:8001") //
+            .build();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        final Message resp = this.rpcResponseFactory.newResponse(CliRequests.GetPeersResponse.getDefaultInstance(),
+            new Status(-1, "failed"));
+        cb.complete(resp, null);
+
+        final Message msg = future.get();
+
+        assertTrue(msg instanceof ErrorResponse);
+        assertEquals(((ErrorResponse) msg).getErrorMsg(), "failed");
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertTrue(!done.status.isOk());
+        assertEquals(done.status.getErrorMsg(), "failed");
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java
new file mode 100644
index 0000000..f1cf165
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.util.AdaptiveBufAllocator;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.RecyclableByteBufferList;
+import com.alipay.sofa.jraft.util.RecycleUtil;
+import com.google.protobuf.ZeroByteStringHelper;
+
+import static com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class AppendEntriesBenchmark {
+
+    /**
+     * entryCount=256, sizeOfEntry=2048
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt  Score   Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  4.139 ± 2.662  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3  0.148 ± 0.027  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  3.730 ± 0.355  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  3.069 ± 3.563  ops/ms
+     *
+     *
+     * entryCount=256, sizeOfEntry=1024
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt  Score   Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  8.290 ± 5.438  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3  0.326 ± 0.137  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  7.559 ± 1.245  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  6.602 ± 0.859  ops/ms
+     *
+     * entryCount=256, sizeOfEntry=512
+     * ---------------------------------------------------------------------------
+     *
+     * Benchmark                                  Mode  Cnt   Score   Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  14.358 ± 8.622  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3   1.625 ± 0.058  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  15.332 ± 1.531  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  12.614 ± 5.904  ops/ms
+     *
+     * entryCount=256, sizeOfEntry=256
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt   Score    Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  32.506 ± 21.961  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3   6.595 ±  5.772  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  27.847 ± 14.010  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  26.427 ±  5.187  ops/ms
+     *
+     * entryCount=256, sizeOfEntry=128
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt   Score    Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  60.014 ± 47.206  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3  22.884 ±  3.286  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  57.373 ±  8.201  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  43.923 ±  7.133  ops/ms
+     *
+     * entryCount=256, sizeOfEntry=64
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt    Score    Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  114.016 ± 84.874  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3   71.699 ± 19.016  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  107.714 ±  7.944  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3   71.767 ± 14.510  ops/ms
+     *
+     * entryCount=256, sizeOfEntry=16
+     * ---------------------------------------------------------------------------
+     * Benchmark                                  Mode  Cnt    Score     Error   Units
+     * AppendEntriesBenchmark.adaptiveAndPooled  thrpt    3  285.386 ± 114.361  ops/ms
+     * AppendEntriesBenchmark.copy               thrpt    3  243.805 ±  31.725  ops/ms
+     * AppendEntriesBenchmark.pooled             thrpt    3  293.779 ±  76.557  ops/ms
+     * AppendEntriesBenchmark.zeroCopy           thrpt    3  124.669 ±  32.460  ops/ms
+     */
+
+    private static final ThreadLocal handleThreadLocal = ThreadLocal
+                                                                                        .withInitial(AdaptiveBufAllocator.DEFAULT::newHandle);
+
+    private int                                                   entryCount;
+    private int                                                   sizeOfEntry;
+
+    @Setup
+    public void setup() {
+        this.entryCount = 256;
+        this.sizeOfEntry = 2048;
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        final int size = ThreadLocalRandom.current().nextInt(100, 1000);
+        System.out.println(sendEntries1(256, size).length);
+        System.out.println(sendEntries2(256, size).length);
+        System.out.println(sendEntries3(256, size, AdaptiveBufAllocator.DEFAULT.newHandle()).length);
+        System.out.println(sendEntries4(256, size).length);
+
+        Options opt = new OptionsBuilder() //
+            .include(AppendEntriesBenchmark.class.getSimpleName()) //
+            .warmupIterations(1) //
+            .warmupTime(TimeValue.seconds(5)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(8) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void copy() {
+        sendEntries1(this.entryCount, this.sizeOfEntry);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void pooled() {
+        sendEntries2(this.entryCount, this.sizeOfEntry);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void adaptiveAndPooled() {
+        sendEntries3(this.entryCount, this.sizeOfEntry, handleThreadLocal.get());
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void zeroCopy() {
+        sendEntries4(this.entryCount, this.sizeOfEntry);
+    }
+
+    private static byte[] sendEntries1(final int entryCount, final int sizeOfEntry) {
+        final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+        fillCommonFields(rb);
+        final ByteBufferCollector dataBuffer = ByteBufferCollector.allocate();
+        for (int i = 0; i < entryCount; i++) {
+            final byte[] bytes = new byte[sizeOfEntry];
+            ThreadLocalRandom.current().nextBytes(bytes);
+            final ByteBuffer buf = ByteBuffer.wrap(bytes);
+            dataBuffer.put(buf.slice());
+        }
+        final ByteBuffer buf = dataBuffer.getBuffer();
+        buf.flip();
+        rb.setData(ZeroByteStringHelper.wrap(buf));
+        return rb.build().toByteArray();
+    }
+
+    private static byte[] sendEntries2(final int entryCount, final int sizeOfEntry) {
+        final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+        fillCommonFields(rb);
+        final ByteBufferCollector dataBuffer = ByteBufferCollector.allocateByRecyclers();
+        try {
+            for (int i = 0; i < entryCount; i++) {
+                final byte[] bytes = new byte[sizeOfEntry];
+                ThreadLocalRandom.current().nextBytes(bytes);
+                final ByteBuffer buf = ByteBuffer.wrap(bytes);
+                dataBuffer.put(buf.slice());
+            }
+            final ByteBuffer buf = dataBuffer.getBuffer();
+            buf.flip();
+            rb.setData(ZeroByteStringHelper.wrap(buf));
+            return rb.build().toByteArray();
+        } finally {
+            RecycleUtil.recycle(dataBuffer);
+        }
+    }
+
+    private static byte[] sendEntries3(final int entryCount, final int sizeOfEntry,
+                                       AdaptiveBufAllocator.Handle allocator) {
+        final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+        fillCommonFields(rb);
+        final ByteBufferCollector dataBuffer = allocator.allocateByRecyclers();
+        try {
+            for (int i = 0; i < entryCount; i++) {
+                final byte[] bytes = new byte[sizeOfEntry];
+                ThreadLocalRandom.current().nextBytes(bytes);
+                final ByteBuffer buf = ByteBuffer.wrap(bytes);
+                dataBuffer.put(buf.slice());
+            }
+            final ByteBuffer buf = dataBuffer.getBuffer();
+            buf.flip();
+            final int remaining = buf.remaining();
+            allocator.record(remaining);
+            rb.setData(ZeroByteStringHelper.wrap(buf));
+            return rb.build().toByteArray();
+        } finally {
+            RecycleUtil.recycle(dataBuffer);
+        }
+    }
+
+    private static byte[] sendEntries4(final int entryCount, final int sizeOfEntry) {
+        final AppendEntriesRequest.Builder rb = AppendEntriesRequest.newBuilder();
+        fillCommonFields(rb);
+        final RecyclableByteBufferList dataBuffer = RecyclableByteBufferList.newInstance();
+        try {
+            for (int i = 0; i < entryCount; i++) {
+                final byte[] bytes = new byte[sizeOfEntry];
+                ThreadLocalRandom.current().nextBytes(bytes);
+                final ByteBuffer buf = ByteBuffer.wrap(bytes);
+                dataBuffer.add(buf.slice());
+            }
+            rb.setData(ZeroByteStringHelper.concatenate(dataBuffer));
+            return rb.build().toByteArray();
+        } finally {
+            RecycleUtil.recycle(dataBuffer);
+        }
+    }
+
+    private static void fillCommonFields(final AppendEntriesRequest.Builder rb) {
+        rb.setTerm(1) //
+            .setGroupId("1") //
+            .setServerId("test") //
+            .setPeerId("127.0.0.1:8080") //
+            .setPrevLogIndex(2) //
+            .setPrevLogTerm(3) //
+            .setCommittedIndex(4);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java
new file mode 100644
index 0000000..92cf3fe
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rpc.impl.PingRequestProcessor;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ConnectionRefreshTest {
+
+    @Ignore
+    @Test
+    public void simulation() throws InterruptedException {
+        ProtobufMsgFactory.load();
+
+        final RpcServer server = RpcFactoryHelper.rpcFactory().createRpcServer(new Endpoint("127.0.0.1", 19991));
+        server.registerProcessor(new PingRequestProcessor());
+        server.init(null);
+
+        final Endpoint target = new Endpoint("my.test.host1.com", 19991);
+
+        final RpcClient client = RpcFactoryHelper.rpcFactory().createRpcClient();
+        client.init(null);
+
+        final RpcRequests.PingRequest req = RpcRequests.PingRequest.newBuilder() //
+            .setSendTimestamp(System.currentTimeMillis()) //
+            .build();
+
+        for (int i = 0; i < 1000; i++) {
+            try {
+                final Object resp = client.invokeSync(target, req, 3000);
+                System.out.println(resp);
+            } catch (final Exception e) {
+                e.printStackTrace();
+            }
+            Thread.sleep(1000);
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactoryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactoryTest.java
new file mode 100644
index 0000000..beded3e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufMsgFactoryTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+
+public class ProtobufMsgFactoryTest {
+    static {
+        ProtobufMsgFactory.load();
+    }
+
+    @Test
+    public void testNewMessageByJavaClassName() {
+        SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(99).setLastIncludedTerm(1).build();
+        SnapshotMeta pMeta = ProtobufMsgFactory
+            .newMessageByJavaClassName(meta.getClass().getName(), meta.toByteArray());
+        assertNotNull(pMeta);
+        assertNotSame(pMeta, meta);
+        assertEquals(pMeta, meta);
+    }
+
+    @Test
+    public void testNewMessage() {
+        SnapshotMeta meta = SnapshotMeta.newBuilder().setLastIncludedIndex(99).setLastIncludedTerm(1).build();
+        SnapshotMeta pMeta = ProtobufMsgFactory.newMessageByProtoClassName("jraft.SnapshotMeta", meta.toByteArray());
+        assertNotNull(pMeta);
+        assertNotSame(pMeta, meta);
+        assertEquals(pMeta, meta);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufSerializerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufSerializerTest.java
new file mode 100644
index 0000000..bad0fd5
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/ProtobufSerializerTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Test;
+
+import com.alipay.remoting.rpc.RpcCommandFactory;
+import com.alipay.remoting.rpc.protocol.RpcRequestCommand;
+import com.alipay.remoting.rpc.protocol.RpcResponseCommand;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequestHeader;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class ProtobufSerializerTest {
+
+    private final ProtobufSerializer serializer = ProtobufSerializer.INSTANCE;
+
+    final RpcCommandFactory          cmdFactory = new RpcCommandFactory();
+
+    @Test
+    public void testEncodeDecodeRequestContent() throws Exception {
+        final PingRequest reqObject = TestUtils.createPingRequest();
+        final RpcRequestCommand request = cmdFactory.createRequestCommand(reqObject);
+        request.setRequestClass(PingRequest.class.getName());
+        assertTrue(serializer.serializeContent(request, null));
+
+        request.setRequestObject(null);
+        assertTrue(serializer.deserializeContent(request));
+        assertNotNull(request.getRequestObject());
+        assertEquals(reqObject, request.getRequestObject());
+        assertNotSame(reqObject, request.getRequestObject());
+    }
+
+    @Test
+    public void testEncodeDecodeAppendEntiresRequestHeader() throws Exception {
+        final AppendEntriesRequest reqObject = AppendEntriesRequest.newBuilder() //
+            .setGroupId("testGroup") //
+            .setPeerId("testPeer")//
+            .setServerId("testServer") //
+            .setTerm(1)//
+            .setPrevLogIndex(1)//
+            .setPrevLogTerm(0) //
+            .setCommittedIndex(1).build();
+        final RpcCommandFactory cmdFactory = new RpcCommandFactory();
+        final RpcRequestCommand request = cmdFactory.createRequestCommand(reqObject);
+        request.setRequestClass(AppendEntriesRequest.class.getName());
+        assertNull(request.getHeader());
+        assertTrue(serializer.serializeContent(request, null));
+        assertTrue(serializer.serializeHeader(request, null));
+        assertNull(request.getRequestHeader());
+
+        request.setRequestObject(null);
+        assertTrue(serializer.deserializeContent(request));
+        assertTrue(serializer.deserializeHeader(request));
+        assertNotNull(request.getRequestObject());
+        assertNotNull(request.getRequestHeader());
+
+        assertEquals(reqObject, request.getRequestObject());
+        assertNotSame(reqObject, request.getRequestObject());
+
+        final AppendEntriesRequestHeader header = (AppendEntriesRequestHeader) request.getRequestHeader();
+        assertEquals("testGroup", header.getGroupId());
+        assertEquals("testPeer", header.getPeerId());
+        assertEquals("testServer", header.getServerId());
+
+    }
+
+    @Test
+    public void testEncodeDecodeResponseContent() throws Exception {
+        final PingRequest reqObject = TestUtils.createPingRequest();
+        final RpcRequestCommand request = cmdFactory.createRequestCommand(reqObject);
+        final ErrorResponse respObject = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null,
+            new Status(-1, "test"));
+        final RpcResponseCommand response = cmdFactory.createResponse(respObject, request);
+        response.setResponseClass(ErrorResponse.class.getName());
+        assertTrue(serializer.serializeContent(response));
+
+        response.setResponseObject(null);
+        assertTrue(serializer.deserializeContent(response, null));
+        assertNotNull(response.getResponseObject());
+        assertEquals(respObject, response.getResponseObject());
+        assertNotSame(respObject, response.getResponseObject());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java
new file mode 100644
index 0000000..7485244
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/RpcResponseFactoryTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+import static org.junit.Assert.assertEquals;
+
+public class RpcResponseFactoryTest {
+    @Test
+    public void testNewResponseFromStatus() {
+        ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, Status.OK());
+        assertEquals(response.getErrorCode(), 0);
+        assertEquals(response.getErrorMsg(), "");
+    }
+
+    @Test
+    public void testNewResponseWithErrorStatus() {
+        ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null,
+            new Status(300, "test"));
+        assertEquals(response.getErrorCode(), 300);
+        assertEquals(response.getErrorMsg(), "test");
+    }
+
+    @Test
+    public void testNewResponseWithVaridicArgs() {
+        ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, 300,
+            "hello %s %d", "world", 99);
+        assertEquals(response.getErrorCode(), 300);
+        assertEquals(response.getErrorMsg(), "hello world 99");
+    }
+
+    @Test
+    public void testNewResponseWithArgs() {
+        ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, 300,
+            "hello world");
+        assertEquals(response.getErrorCode(), 300);
+        assertEquals(response.getErrorMsg(), "hello world");
+    }
+
+    @Test
+    public void testNewResponseWithRaftError() {
+        ErrorResponse response = (ErrorResponse) RpcFactoryHelper.responseFactory().newResponse(null, RaftError.EAGAIN,
+            "hello world");
+        assertEquals(response.getErrorCode(), RaftError.EAGAIN.getNumber());
+        assertEquals(response.getErrorMsg(), "hello world");
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java
new file mode 100644
index 0000000..ebd2b2a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/FutureTest.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.io.IOException;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class FutureTest {
+
+    private static final Logger log = LoggerFactory.getLogger(FutureImpl.class);
+
+    private static final class NotifyFutureRunner implements Runnable {
+        FutureImpl future;
+        long                sleepTime;
+        Throwable           throwable;
+
+        public NotifyFutureRunner(FutureImpl future, long sleepTime, Throwable throwable) {
+            super();
+            this.future = future;
+            this.sleepTime = sleepTime;
+            this.throwable = throwable;
+        }
+
+        @Override
+        public void run() {
+            try {
+                Thread.sleep(this.sleepTime);
+                if (this.throwable != null) {
+                    this.future.failure(this.throwable);
+                } else {
+                    this.future.setResult(true);
+                }
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+    }
+
+    @Test
+    public void testGet() throws Exception {
+        FutureImpl future = new FutureImpl();
+        new Thread(new NotifyFutureRunner(future, 2000, null)).start();
+        boolean result = future.get();
+        assertTrue(result);
+        assertTrue(future.isDone());
+        assertFalse(future.isCancelled());
+    }
+
+    @Test
+    public void testGetImmediately() throws Exception {
+        FutureImpl future = new FutureImpl();
+        future.setResult(true);
+        boolean result = future.get();
+        assertTrue(result);
+        assertTrue(future.isDone());
+        assertFalse(future.isCancelled());
+    }
+
+    @Test
+    public void testGetException() throws Exception {
+        FutureImpl future = new FutureImpl();
+        new Thread(new NotifyFutureRunner(future, 2000, new IOException("hello"))).start();
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertEquals("hello", e.getCause().getMessage());
+
+        }
+        assertTrue(future.isDone());
+        assertFalse(future.isCancelled());
+
+    }
+
+    @Test
+    public void testCancel() throws Exception {
+        final FutureImpl future = new FutureImpl();
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                try {
+                    Thread.sleep(3000);
+                    future.cancel(true);
+                } catch (Exception e) {
+                    log.error(e.getMessage(), e);
+                }
+            }
+        }).start();
+        try {
+            future.get();
+            fail();
+        } catch (CancellationException e) {
+            assertTrue(true);
+
+        }
+        assertTrue(future.isDone());
+        assertTrue(future.isCancelled());
+    }
+
+    @Test
+    public void testGetTimeout() throws Exception {
+        FutureImpl future = new FutureImpl();
+        try {
+            future.get(1000, TimeUnit.MILLISECONDS);
+            fail();
+        } catch (TimeoutException e) {
+            assertTrue(true);
+        }
+    }
+}
\ No newline at end of file
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
new file mode 100644
index 0000000..e7d137e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+
+public class PingRequestProcessorTest {
+
+    @Test
+    public void testHandlePing() throws Exception {
+        PingRequestProcessor processor = new PingRequestProcessor();
+        MockAsyncContext ctx = new MockAsyncContext();
+        processor.handleRequest(ctx, TestUtils.createPingRequest());
+        ErrorResponse response = (ErrorResponse) ctx.getResponseObject();
+        assertEquals(0, response.getErrorCode());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java
new file mode 100644
index 0000000..c1217bb
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AbstractCliRequestProcessorTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.google.protobuf.Message;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public abstract class AbstractCliRequestProcessorTest {
+    @Mock
+    private Node               node;
+    private final String       groupId   = "test";
+    private final String       peerIdStr = "localhost:8081";
+    protected MockAsyncContext asyncContext;
+
+    public abstract T createRequest(String groupId, PeerId peerId);
+
+    public abstract BaseCliRequestProcessor newProcessor();
+
+    public abstract void verify(String interest, Node node, ArgumentCaptor doneArg);
+
+    public void mockNodes(final int n) {
+        ArrayList peers = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            peers.add(JRaftUtils.getPeerId("localhost:" + (8081 + i)));
+        }
+        List learners = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            learners.add(JRaftUtils.getPeerId("learner:" + (8081 + i)));
+        }
+        Mockito.when(this.node.listPeers()).thenReturn(peers);
+        Mockito.when(this.node.listLearners()).thenReturn(learners);
+    }
+
+    @Before
+    public void setup() {
+        this.asyncContext = new MockAsyncContext();
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testHandleRequest() {
+        this.mockNodes(3);
+        Mockito.when(this.node.getGroupId()).thenReturn(this.groupId);
+        PeerId peerId = new PeerId();
+        peerId.parse(this.peerIdStr);
+        Mockito.when(this.node.getOptions()).thenReturn(new NodeOptions());
+        Mockito.when(this.node.getNodeId()).thenReturn(new NodeId("test", peerId));
+        NodeManager.getInstance().addAddress(peerId.getEndpoint());
+        NodeManager.getInstance().add(this.node);
+
+        BaseCliRequestProcessor processor = newProcessor();
+        processor.handleRequest(this.asyncContext, createRequest(this.groupId, peerId));
+        ArgumentCaptor doneArg = ArgumentCaptor.forClass(Closure.class);
+        verify(processor.interest(), this.node, doneArg);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessorTest.java
new file mode 100644
index 0000000..8b1cd14
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddLearnersRequestProcessorTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+import java.util.Arrays;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.AddLearnersRequest;
+import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse;
+
+public class AddLearnersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public AddLearnersRequest createRequest(final String groupId, final PeerId peerId) {
+        return AddLearnersRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            addAllLearners(Arrays.asList("learner:8082", "test:8182", "test:8183")).build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new AddLearnersRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(final String interest, final Node node, final ArgumentCaptor doneArg) {
+        assertEquals(interest, AddLearnersRequest.class.getName());
+        Mockito.verify(node).addLearners(
+            eq(Arrays.asList(new PeerId("learner", 8082), new PeerId("test", 8182), new PeerId("test", 8183))),
+            doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[learner:8081, learner:8082, learner:8083]", this.asyncContext.as(LearnersOpResponse.class)
+            .getOldLearnersList().toString());
+        assertEquals("[learner:8081, learner:8082, learner:8083, test:8182, test:8183]",
+            this.asyncContext.as(LearnersOpResponse.class).getNewLearnersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessorTest.java
new file mode 100644
index 0000000..67f0682
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/AddPeerRequestProcessorTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerRequest;
+import com.alipay.sofa.jraft.rpc.CliRequests.AddPeerResponse;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+public class AddPeerRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public AddPeerRequest createRequest(String groupId, PeerId peerId) {
+        return AddPeerRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            setPeerId("test:8181").build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new AddPeerRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(interest, AddPeerRequest.class.getName());
+        Mockito.verify(node).addPeer(eq(new PeerId("test", 8181)), doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[localhost:8081, localhost:8082, localhost:8083]", this.asyncContext.as(AddPeerResponse.class)
+            .getOldPeersList().toString());
+        assertEquals("[localhost:8081, localhost:8082, localhost:8083, test:8181]",
+            this.asyncContext.as(AddPeerResponse.class).getNewPeersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessorTest.java
new file mode 100644
index 0000000..ef2f2df
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/BaseCliRequestProcessorTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertSame;
+
+@RunWith(MockitoJUnitRunner.class)
+public class BaseCliRequestProcessorTest {
+    private static class MockCliRequestProcessor extends BaseCliRequestProcessor {
+
+        private String            peerId;
+        private String            groupId;
+        private RpcRequestClosure done;
+        private CliRequestContext ctx;
+
+        public MockCliRequestProcessor(String peerId, String groupId) {
+            super(null, null);
+            this.peerId = peerId;
+            this.groupId = groupId;
+        }
+
+        @Override
+        protected String getPeerId(PingRequest request) {
+            return this.peerId;
+        }
+
+        @Override
+        protected String getGroupId(PingRequest request) {
+            return this.groupId;
+        }
+
+        @Override
+        protected Message processRequest0(CliRequestContext ctx, PingRequest request, RpcRequestClosure done) {
+            this.ctx = ctx;
+            this.done = done;
+            return RpcFactoryHelper.responseFactory().newResponse(null, Status.OK());
+        }
+
+        @Override
+        public String interest() {
+            return PingRequest.class.getName();
+        }
+
+    }
+
+    private MockCliRequestProcessor processor;
+    private PeerId                  peer;
+    private MockAsyncContext        asyncContext;
+
+    @Before
+    public void setup() {
+        this.asyncContext = new MockAsyncContext();
+        this.peer = JRaftUtils.getPeerId("localhost:8081");
+        this.processor = new MockCliRequestProcessor(this.peer.toString(), "test");
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testOK() {
+        Node node = mockNode(false);
+
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(this.processor.done);
+        assertSame(this.processor.ctx.node, node);
+        assertNotNull(resp);
+        assertEquals(0, resp.getErrorCode());
+    }
+
+    @Test
+    public void testDisableCli() {
+        mockNode(true);
+
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.EACCES.getNumber(), resp.getErrorCode());
+        assertEquals("Cli service is not allowed to access node ", resp.getErrorMsg());
+    }
+
+    private Node mockNode(boolean disableCli) {
+        Node node = Mockito.mock(Node.class);
+        Mockito.when(node.getGroupId()).thenReturn("test");
+        Mockito.when(node.getNodeId()).thenReturn(new NodeId("test", this.peer.copy()));
+        NodeOptions opts = new NodeOptions();
+        opts.setDisableCli(disableCli);
+        Mockito.when(node.getOptions()).thenReturn(opts);
+        NodeManager.getInstance().addAddress(this.peer.getEndpoint());
+        NodeManager.getInstance().add(node);
+        return node;
+    }
+
+    @Test
+    public void testInvalidPeerId() {
+        this.processor = new MockCliRequestProcessor("localhost", "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
+        assertEquals("Fail to parse peer: localhost", resp.getErrorMsg());
+    }
+
+    @Test
+    public void testEmptyNodes() {
+        this.processor = new MockCliRequestProcessor(null, "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.ENOENT.getNumber(), resp.getErrorCode());
+        assertEquals("Empty nodes in group test", resp.getErrorMsg());
+    }
+
+    @Test
+    public void testManyNodes() {
+        Node node1 = Mockito.mock(Node.class);
+        Mockito.when(node1.getGroupId()).thenReturn("test");
+        Mockito.when(node1.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8081)));
+        NodeOptions opts = new NodeOptions();
+        Mockito.when(node1.getOptions()).thenReturn(opts);
+        NodeManager.getInstance().addAddress(new Endpoint("localhost", 8081));
+        NodeManager.getInstance().add(node1);
+
+        Node node2 = Mockito.mock(Node.class);
+        Mockito.when(node2.getGroupId()).thenReturn("test");
+        Mockito.when(node2.getNodeId()).thenReturn(new NodeId("test", new PeerId("localhost", 8082)));
+        Mockito.when(node2.getOptions()).thenReturn(opts);
+        NodeManager.getInstance().addAddress(new Endpoint("localhost", 8082));
+        NodeManager.getInstance().add(node2);
+
+        this.processor = new MockCliRequestProcessor(null, "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
+        assertEquals("Peer must be specified since there're 2 nodes in group test", resp.getErrorMsg());
+    }
+
+    @Test
+    public void testSingleNode() {
+        Node node = this.mockNode(false);
+        this.processor = new MockCliRequestProcessor(null, "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertSame(this.processor.ctx.node, node);
+        assertNotNull(resp);
+        assertEquals(0, resp.getErrorCode());
+    }
+
+    @Test
+    public void testPeerIdNotFound() {
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.ENOENT.getNumber(), resp.getErrorCode());
+        assertEquals("Fail to find node localhost:8081 in group test", resp.getErrorMsg());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessorTest.java
new file mode 100644
index 0000000..8152f0e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ChangePeersRequestProcessorTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersRequest;
+import com.alipay.sofa.jraft.rpc.CliRequests.ChangePeersResponse;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+public class ChangePeersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public ChangePeersRequest createRequest(String groupId, PeerId peerId) {
+        return ChangePeersRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            addNewPeers("localhost:8084").addNewPeers("localhost:8085").build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new ChangePeersRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(interest, ChangePeersRequest.class.getName());
+        Mockito.verify(node).changePeers(eq(JRaftUtils.getConfiguration("localhost:8084,localhost:8085")),
+            doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[localhost:8081, localhost:8082, localhost:8083]", this.asyncContext
+            .as(ChangePeersResponse.class).getOldPeersList().toString());
+        assertEquals("[localhost:8084, localhost:8085]", this.asyncContext.as(ChangePeersResponse.class)
+            .getNewPeersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessorTest.java
new file mode 100644
index 0000000..3ffc5d8
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/GetPeersRequestProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import org.mockito.ArgumentCaptor;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersRequest;
+import com.alipay.sofa.jraft.rpc.CliRequests.GetPeersResponse;
+
+public class GetPeersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public GetPeersRequest createRequest(final String groupId, final PeerId peerId) {
+        return GetPeersRequest.newBuilder(). //
+            setGroupId(groupId). //
+            build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new GetPeersRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(final String interest, final Node node, final ArgumentCaptor doneArg) {
+        assertEquals(interest, GetPeersRequest.class.getName());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[localhost:8081, localhost:8082, localhost:8083]", this.asyncContext.as(GetPeersResponse.class)
+            .getPeersList().toString());
+        assertEquals("[learner:8081, learner:8082, learner:8083]", this.asyncContext.as(GetPeersResponse.class)
+            .getLearnersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessorTest.java
new file mode 100644
index 0000000..eace0f8
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemoveLearnersRequestProcessorTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+import java.util.Arrays;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse;
+import com.alipay.sofa.jraft.rpc.CliRequests.RemoveLearnersRequest;
+
+public class RemoveLearnersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public RemoveLearnersRequest createRequest(final String groupId, final PeerId peerId) {
+        return RemoveLearnersRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            addAllLearners(Arrays.asList("learner:8082", "test:8183")).build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new RemoveLearnersRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(final String interest, final Node node, final ArgumentCaptor doneArg) {
+        assertEquals(interest, RemoveLearnersRequest.class.getName());
+        Mockito.verify(node).removeLearners(eq(Arrays.asList(new PeerId("learner", 8082), new PeerId("test", 8183))),
+            doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[learner:8081, learner:8082, learner:8083]", this.asyncContext.as(LearnersOpResponse.class)
+            .getOldLearnersList().toString());
+        assertEquals("[learner:8081, learner:8083]", this.asyncContext.as(LearnersOpResponse.class)
+            .getNewLearnersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessorTest.java
new file mode 100644
index 0000000..2b01f22
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/RemovePeerRequestProcessorTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerRequest;
+import com.alipay.sofa.jraft.rpc.CliRequests.RemovePeerResponse;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+public class RemovePeerRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public RemovePeerRequest createRequest(String groupId, PeerId peerId) {
+        return RemovePeerRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            setPeerId("localhost:8082").build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new RemovePeerRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(interest, RemovePeerRequest.class.getName());
+        Mockito.verify(node).removePeer(eq(new PeerId("localhost", 8082)), doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[localhost:8081, localhost:8082, localhost:8083]", this.asyncContext.as(RemovePeerResponse.class)
+            .getOldPeersList().toString());
+        assertEquals("[localhost:8081, localhost:8083]", this.asyncContext.as(RemovePeerResponse.class)
+            .getNewPeersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessorTest.java
new file mode 100644
index 0000000..9928b66
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetLearnersRequestProcessorTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+
+import java.util.Arrays;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.LearnersOpResponse;
+import com.alipay.sofa.jraft.rpc.CliRequests.ResetLearnersRequest;
+
+public class ResetLearnersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public ResetLearnersRequest createRequest(final String groupId, final PeerId peerId) {
+        return ResetLearnersRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setLeaderId(peerId.toString()). //
+            addAllLearners(Arrays.asList("learner:8082", "test:8182", "test:8183")).build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new ResetLearnersRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(final String interest, final Node node, final ArgumentCaptor doneArg) {
+        assertEquals(interest, ResetLearnersRequest.class.getName());
+        Mockito.verify(node).resetLearners(
+            eq(Arrays.asList(new PeerId("learner", 8082), new PeerId("test", 8182), new PeerId("test", 8183))),
+            doneArg.capture());
+        Closure done = doneArg.getValue();
+        assertNotNull(done);
+        done.run(Status.OK());
+        assertNotNull(this.asyncContext.getResponseObject());
+        assertEquals("[learner:8081, learner:8082, learner:8083]", this.asyncContext.as(LearnersOpResponse.class)
+            .getOldLearnersList().toString());
+        assertEquals("[learner:8082, test:8182, test:8183]", this.asyncContext.as(LearnersOpResponse.class)
+            .getNewLearnersList().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeersRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeersRequestProcessorTest.java
new file mode 100644
index 0000000..706d41e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/ResetPeersRequestProcessorTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.ResetPeerRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ResetPeersRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public ResetPeerRequest createRequest(String groupId, PeerId peerId) {
+        return ResetPeerRequest.newBuilder(). //
+            setGroupId(groupId). //
+            setPeerId(peerId.toString()). //
+            addNewPeers("localhost:8084").addNewPeers("localhost:8085").build();
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new ResetPeerRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(interest, ResetPeerRequest.class.getName());
+        Mockito.verify(node).resetPeers(JRaftUtils.getConfiguration("localhost:8084,localhost:8085"));
+        assertNotNull(asyncContext.getResponseObject());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessorTest.java
new file mode 100644
index 0000000..4014391
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/SnapshotRequestProcessorTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.SnapshotRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class SnapshotRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public SnapshotRequest createRequest(String groupId, PeerId peerId) {
+        return SnapshotRequest.newBuilder().setGroupId(groupId).setPeerId(peerId.toString()).build();
+
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new SnapshotRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(SnapshotRequest.class.getName(), interest);
+        Mockito.verify(node).snapshot(doneArg.capture());
+        assertNotNull(doneArg.getValue());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeadershipRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeadershipRequestProcessorTest.java
new file mode 100644
index 0000000..de42350
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/cli/TransferLeadershipRequestProcessorTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.cli;
+
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.CliRequests.TransferLeaderRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class TransferLeadershipRequestProcessorTest extends AbstractCliRequestProcessorTest {
+
+    @Override
+    public TransferLeaderRequest createRequest(String groupId, PeerId peerId) {
+        return TransferLeaderRequest.newBuilder().setGroupId(groupId).setLeaderId(peerId.toString())
+            .setPeerId("localhost:8082").build();
+
+    }
+
+    @Override
+    public BaseCliRequestProcessor newProcessor() {
+        return new TransferLeaderRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, Node node, ArgumentCaptor doneArg) {
+        assertEquals(TransferLeaderRequest.class.getName(), interest);
+        Mockito.verify(node).transferLeadershipTo(new PeerId("localhost", 8082));
+        assertNotNull(asyncContext.getResponseObject());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java
new file mode 100644
index 0000000..e716b19
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.rpc.impl.core.AppendEntriesRequestProcessor.PeerPair;
+import com.alipay.sofa.jraft.rpc.impl.core.AppendEntriesRequestProcessor.PeerRequestContext;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.concurrent.ConcurrentHashSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.mockito.Matchers.eq;
+
+public class AppendEntriesRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private AppendEntriesRequest request;
+
+    private final String         serverId = "localhost:8082";
+
+    @Override
+    public AppendEntriesRequest createRequest(final String groupId, final PeerId peerId) {
+        this.request = AppendEntriesRequest.newBuilder().setCommittedIndex(0). //
+            setGroupId(groupId). //
+            setPeerId(peerId.toString()).//
+            setServerId(this.serverId). //
+            setPrevLogIndex(0). //
+            setTerm(0). //
+            setPrevLogTerm(0).build();
+        return this.request;
+    }
+
+    @Mock
+    private Connection conn;
+
+    @Override
+    public void setup() {
+        super.setup();
+        this.asyncContext = new MockAsyncContext() {
+            @Override
+            public Connection getConnection() {
+                return AppendEntriesRequestProcessorTest.this.conn;
+            }
+        };
+        Set pairs = new ConcurrentHashSet<>();
+        pairs.add(new PeerPair(this.peerIdStr, this.serverId));
+        Mockito.when(this.conn.getAttribute(AppendEntriesRequestProcessor.PAIR_ATTR)).thenReturn(pairs);
+    }
+
+    private ExecutorService executor;
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        this.executor = Executors.newSingleThreadExecutor();
+        return new AppendEntriesRequestProcessor(this.executor);
+    }
+
+    @Override
+    public void teardown() {
+        super.teardown();
+        if (this.executor != null) {
+            this.executor.shutdownNow();
+        }
+    }
+
+    @Test
+    public void testPairOf() {
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+
+        PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        assertEquals(pair.remote, this.serverId);
+        assertEquals(pair.local, this.peerIdStr);
+
+        // test constant pool
+        assertSame(pair, processor.pairOf(this.peerIdStr, this.serverId));
+        assertSame(pair, processor.pairOf(this.peerIdStr, this.serverId));
+        assertEquals("PeerPair[localhost:8081 -> localhost:8082]", pair.toString());
+    }
+
+    @Test
+    public void testOnClosed() {
+        mockNode();
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+
+        PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+        assertSame(ctx, processor.getPeerRequestContext(this.groupId, pair));
+        assertSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+
+        processor.onClosed(null, this.conn);
+        assertNull(processor.getPeerRequestContext(this.groupId, pair));
+        assertNotSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+    }
+
+    @Override
+    public void verify(final String interest, final RaftServerService service,
+                       final NodeRequestProcessor processor) {
+        assertEquals(interest, AppendEntriesRequest.class.getName());
+        Mockito.verify(service).handleAppendEntriesRequest(eq(this.request), Mockito.any());
+        final PeerPair pair = ((AppendEntriesRequestProcessor) processor).pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = ((AppendEntriesRequestProcessor) processor).getOrCreatePeerRequestContext(
+            this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+    }
+
+    @Test
+    public void testGetPeerRequestContextRemovePeerRequestContext() {
+        mockNode();
+
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+        assertSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+        assertEquals(0, ctx.getNextRequiredSequence());
+        assertEquals(0, ctx.getAndIncrementSequence());
+        assertEquals(1, ctx.getAndIncrementSequence());
+        assertEquals(0, ctx.getAndIncrementNextRequiredSequence());
+        assertEquals(1, ctx.getAndIncrementNextRequiredSequence());
+        assertFalse(ctx.hasTooManyPendingResponses());
+
+        processor.removePeerRequestContext(this.groupId, pair);
+        final PeerRequestContext newCtx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(newCtx);
+        assertNotSame(ctx, newCtx);
+
+        assertEquals(0, newCtx.getNextRequiredSequence());
+        assertEquals(0, newCtx.getAndIncrementSequence());
+        assertEquals(1, newCtx.getAndIncrementSequence());
+        assertEquals(0, newCtx.getAndIncrementNextRequiredSequence());
+        assertEquals(1, newCtx.getAndIncrementNextRequiredSequence());
+        assertFalse(newCtx.hasTooManyPendingResponses());
+    }
+
+    @Test
+    public void testSendSequenceResponse() {
+        mockNode();
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        final PingRequest msg = TestUtils.createPingRequest();
+        final RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        processor.sendSequenceResponse(this.groupId, pair, 1, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.never()).sendResponse(msg);
+
+        processor.sendSequenceResponse(this.groupId, pair, 0, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.times(2)).sendResponse(msg);
+    }
+
+    @Test
+    public void testTooManyPendingResponses() {
+        final PeerId peer = mockNode();
+        NodeManager.getInstance().get(this.groupId, peer).getRaftOptions().setMaxReplicatorInflightMsgs(2);
+
+        final RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PingRequest msg = TestUtils.createPingRequest();
+        final Connection conn = Mockito.mock(Connection.class);
+        Mockito.when(asyncContext.getConnection()).thenReturn(conn);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, conn);
+        assertNotNull(ctx);
+        processor.sendSequenceResponse(this.groupId, pair, 1, asyncContext, msg);
+        processor.sendSequenceResponse(this.groupId, pair, 2, asyncContext, msg);
+        processor.sendSequenceResponse(this.groupId, pair, 3, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.never()).sendResponse(msg);
+        Mockito.verify(conn).close();
+
+        final PeerRequestContext newCtx = processor.getOrCreatePeerRequestContext(this.groupId, pair, conn);
+        assertNotNull(newCtx);
+        assertNotSame(ctx, newCtx);
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java
new file mode 100644
index 0000000..c0161e7
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.google.protobuf.Message;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public abstract class BaseNodeRequestProcessorTest {
+    @Mock(extraInterfaces = { RaftServerService.class })
+    private Node               node;
+    protected final String     groupId   = "test";
+    protected final String     peerIdStr = "localhost:8081";
+    protected MockAsyncContext asyncContext;
+
+    public abstract T createRequest(String groupId, PeerId peerId);
+
+    public abstract NodeRequestProcessor newProcessor();
+
+    public abstract void verify(String interest, RaftServerService service, NodeRequestProcessor processor);
+
+    @Before
+    public void setup() {
+        Mockito.when(node.getRaftOptions()).thenReturn(new RaftOptions());
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testHandleRequest() {
+        final PeerId peerId = mockNode();
+
+        final NodeRequestProcessor processor = newProcessor();
+        processor.handleRequest(asyncContext, createRequest(groupId, peerId));
+        verify(processor.interest(), (RaftServerService) this.node, processor);
+    }
+
+    protected PeerId mockNode() {
+        Mockito.when(node.getGroupId()).thenReturn(this.groupId);
+        final PeerId peerId = new PeerId();
+        peerId.parse(this.peerIdStr);
+        Mockito.when(node.getNodeId()).thenReturn(new NodeId(groupId, peerId));
+        NodeManager.getInstance().addAddress(peerId.getEndpoint());
+        NodeManager.getInstance().add(node);
+        return peerId;
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java
new file mode 100644
index 0000000..b788cc9
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.ReplicatorGroup;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class DefaultRaftClientServiceTest {
+    private DefaultRaftClientService clientService;
+    @Mock
+    private ReplicatorGroup          rgGroup;
+
+    private final Endpoint           endpoint = new Endpoint("localhost", 8081);
+
+    @Before
+    public void setup() {
+        this.clientService = new DefaultRaftClientService(this.rgGroup);
+        this.clientService.init(new NodeOptions());
+    }
+
+    @Test
+    public void testPreVote() {
+        this.clientService.preVote(this.endpoint, RequestVoteRequest.newBuilder(). //
+            setGroupId("test"). //
+            setLastLogIndex(1). //
+            setLastLogTerm(1). //
+            setPeerId("localhost:1010"). //
+            setTerm(1).setServerId("localhost:1011").setPreVote(true).build(), null);
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java
new file mode 100644
index 0000000..28a7259
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class InstallSnapshotRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private InstallSnapshotRequest request;
+
+    @Override
+    public InstallSnapshotRequest createRequest(String groupId, PeerId peerId) {
+        request = InstallSnapshotRequest.newBuilder().setGroupId(groupId)
+            . //
+            setServerId("localhostL8082")
+            . //
+            setPeerId(peerId.toString())
+            . //
+            setTerm(0)
+            . //
+            setMeta(SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(1).build()).setUri("test")
+            .build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new InstallSnapshotRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service,
+                       NodeRequestProcessor processor) {
+        assertEquals(interest, InstallSnapshotRequest.class.getName());
+        Mockito.verify(service).handleInstallSnapshot(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java
new file mode 100644
index 0000000..38519ee
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.withSettings;
+
+public class NodeRequestProcessorTest {
+
+    private static class MockRequestProcessor extends NodeRequestProcessor {
+
+        private String peerId;
+        private String groupId;
+
+        public MockRequestProcessor(String peerId, String groupId) {
+            super(null, null);
+            this.peerId = peerId;
+            this.groupId = groupId;
+        }
+
+        @Override
+        protected String getPeerId(PingRequest request) {
+            return this.peerId;
+        }
+
+        @Override
+        protected String getGroupId(PingRequest request) {
+            return this.groupId;
+        }
+
+        @Override
+        protected Message processRequest0(RaftServerService serviceService, PingRequest request, RpcRequestClosure done) {
+            return RpcFactoryHelper.responseFactory().newResponse(null, Status.OK());
+        }
+
+        @Override
+        public String interest() {
+            return PingRequest.class.getName();
+        }
+
+    }
+
+    private MockRequestProcessor processor;
+    private MockAsyncContext     asyncContext;
+
+    @Before
+    public void setup() {
+        this.asyncContext = new MockAsyncContext();
+        this.processor = new MockRequestProcessor("localhost:8081", "test");
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testOK() {
+        Node node = Mockito.mock(Node.class, withSettings().extraInterfaces(RaftServerService.class));
+        Mockito.when(node.getGroupId()).thenReturn("test");
+        PeerId peerId = new PeerId("localhost", 8081);
+        Mockito.when(node.getNodeId()).thenReturn(new NodeId("test", peerId));
+        NodeManager.getInstance().addAddress(peerId.getEndpoint());
+        NodeManager.getInstance().add(node);
+
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(0, resp.getErrorCode());
+    }
+
+    @Test
+    public void testInvalidPeerId() {
+        this.processor = new MockRequestProcessor("localhost", "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
+        assertEquals("Fail to parse peerId: localhost", resp.getErrorMsg());
+    }
+
+    @Test
+    public void testPeerIdNotFound() {
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.ENOENT.getNumber(), resp.getErrorCode());
+        assertEquals("Peer id not found: localhost:8081, group: test", resp.getErrorMsg());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java
new file mode 100644
index 0000000..148adc5
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class PreVoteRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private RequestVoteRequest request;
+
+    @Override
+    public RequestVoteRequest createRequest(String groupId, PeerId peerId) {
+        request = RequestVoteRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0). //
+            setLastLogIndex(0). //
+            setLastLogTerm(0). //
+            setPreVote(false).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new RequestVoteRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, RequestVoteRequest.class.getName());
+        Mockito.verify(service).handleRequestVoteRequest(eq(request));
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java
new file mode 100644
index 0000000..9c52f45
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class ReadIndexRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private ReadIndexRequest request;
+
+    @Override
+    public ReadIndexRequest createRequest(String groupId, PeerId peerId) {
+        request = ReadIndexRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new ReadIndexRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, ReadIndexRequest.class.getName());
+        Mockito.verify(service).handleReadIndexRequest(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java
new file mode 100644
index 0000000..a7580b9
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class RequestVoteRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private RequestVoteRequest request;
+
+    @Override
+    public RequestVoteRequest createRequest(String groupId, PeerId peerId) {
+        request = RequestVoteRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0). //
+            setLastLogIndex(0). //
+            setLastLogTerm(0). //
+            setPreVote(true).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new RequestVoteRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, RequestVoteRequest.class.getName());
+        Mockito.verify(service).handlePreVoteRequest(eq(request));
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java
new file mode 100644
index 0000000..daa058f
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class TimeoutNowRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private TimeoutNowRequest request;
+
+    @Override
+    public TimeoutNowRequest createRequest(String groupId, PeerId peerId) {
+        request = TimeoutNowRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new TimeoutNowRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, TimeoutNowRequest.class.getName());
+        Mockito.verify(service).handleTimeoutNowRequest(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/BaseStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/BaseStorageTest.java
new file mode 100644
index 0000000..e7f00c1
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/BaseStorageTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+
+import com.alipay.sofa.jraft.test.TestUtils;
+
+public class BaseStorageTest {
+    protected String path;
+
+    public void setup() throws Exception {
+        this.path = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.path));
+    }
+
+    @After
+    public void teardown() throws Exception {
+        FileUtils.deleteDirectory(new File(this.path));
+    }
+
+    protected String writeData() throws IOException {
+        File file = new File(this.path + File.separator + "data");
+        String data = "jraft is great!";
+        FileUtils.writeStringToFile(file, data);
+        return data;
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/FileServiceTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/FileServiceTest.java
new file mode 100644
index 0000000..5ce2726
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/FileServiceTest.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.storage.io.LocalDirReader;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class FileServiceTest {
+
+    private String         path;
+    private LocalDirReader fileReader;
+
+    @Before
+    public void setup() throws Exception {
+        this.path = TestUtils.mkTempDir();
+        this.fileReader = new LocalDirReader(path);
+    }
+
+    @After
+    public void teardown() {
+        FileUtils.deleteQuietly(new File(this.path));
+        FileService.getInstance().clear();
+    }
+
+    @Test
+    public void testAddRemove() {
+        long readerId = FileService.getInstance().addReader(this.fileReader);
+        assertTrue(readerId > 0);
+        assertTrue(FileService.getInstance().removeReader(readerId));
+    }
+
+    @Test
+    public void testGetFileNotFoundReader() {
+        RpcRequests.GetFileRequest request = RpcRequests.GetFileRequest.newBuilder().setCount(Integer.MAX_VALUE)
+            .setFilename("data").setOffset(0).setReaderId(1).build();
+        RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        Message msg = FileService.getInstance().handleGetFile(request, new RpcRequestClosure(asyncContext));
+        assertTrue(msg instanceof RpcRequests.ErrorResponse);
+        RpcRequests.ErrorResponse response = (RpcRequests.ErrorResponse) msg;
+        Assert.assertEquals(RaftError.ENOENT.getNumber(), response.getErrorCode());
+        assertEquals("Fail to find reader=1", response.getErrorMsg());
+    }
+
+    @Test
+    public void testGetFileNotFound() {
+        long readerId = FileService.getInstance().addReader(this.fileReader);
+        RpcRequests.GetFileRequest request = RpcRequests.GetFileRequest.newBuilder().setCount(Integer.MAX_VALUE)
+            .setFilename("data").setOffset(0).setReaderId(readerId).build();
+        RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        Message msg = FileService.getInstance().handleGetFile(request, new RpcRequestClosure(asyncContext));
+        assertTrue(msg instanceof RpcRequests.ErrorResponse);
+        RpcRequests.ErrorResponse response = (RpcRequests.ErrorResponse) msg;
+        assertEquals(RaftError.EIO.getNumber(), response.getErrorCode());
+        assertEquals(String.format("Fail to read from path=%s filename=data", this.path), response.getErrorMsg());
+    }
+
+    private String writeData() throws IOException {
+        File file = new File(this.path + File.separator + "data");
+        String data = "jraft is great!";
+        FileUtils.writeStringToFile(file, data);
+        return data;
+    }
+
+    @Test
+    public void testGetFileData() throws IOException {
+        writeData();
+        long readerId = FileService.getInstance().addReader(this.fileReader);
+        RpcRequests.GetFileRequest request = RpcRequests.GetFileRequest.newBuilder().setCount(Integer.MAX_VALUE)
+            .setFilename("data").setOffset(0).setReaderId(readerId).build();
+        RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        Message msg = FileService.getInstance().handleGetFile(request, new RpcRequestClosure(asyncContext));
+        assertTrue(msg instanceof RpcRequests.GetFileResponse);
+        RpcRequests.GetFileResponse response = (RpcRequests.GetFileResponse) msg;
+        assertTrue(response.getEof());
+        assertEquals("jraft is great!", new String(response.getData().toByteArray()));
+        assertEquals(-1, response.getReadSize());
+    }
+
+    private String writeLargeData() throws IOException {
+        File file = new File(this.path + File.separator + "data");
+        String data = "jraft is great!";
+        for (int i = 0; i < 1000; i++) {
+            FileUtils.writeStringToFile(file, data, true);
+        }
+        return data;
+    }
+
+    @Test
+    public void testGetLargeFileData() throws IOException {
+        final String data = writeLargeData();
+        final long readerId = FileService.getInstance().addReader(this.fileReader);
+        int fileOffset = 0;
+        while (true) {
+            final RpcRequests.GetFileRequest request = RpcRequests.GetFileRequest.newBuilder() //
+                .setCount(4096).setFilename("data") //
+                .setOffset(fileOffset) //
+                .setReaderId(readerId) //
+                .build();
+            final RpcContext asyncContext = Mockito.mock(RpcContext.class);
+            final Message msg = FileService.getInstance() //
+                .handleGetFile(request, new RpcRequestClosure(asyncContext));
+            assertTrue(msg instanceof RpcRequests.GetFileResponse);
+            final RpcRequests.GetFileResponse response = (RpcRequests.GetFileResponse) msg;
+            final byte[] sourceArray = data.getBytes();
+            final byte[] respData = response.getData().toByteArray();
+            final int length = sourceArray.length;
+            int offset = 0;
+            while (offset + length <= respData.length) {
+                final byte[] respArray = new byte[length];
+                System.arraycopy(respData, offset, respArray, 0, length);
+                assertArrayEquals(sourceArray, respArray);
+                offset += length;
+            }
+            fileOffset += offset;
+            if (response.getEof()) {
+                break;
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/SnapshotExecutorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/SnapshotExecutorTest.java
new file mode 100644
index 0000000..a6967fa
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/SnapshotExecutorTest.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.LoadSnapshotClosure;
+import com.alipay.sofa.jraft.closure.SaveSnapshotClosure;
+import com.alipay.sofa.jraft.closure.SynchronizedClosure;
+import com.alipay.sofa.jraft.core.DefaultJRaftServiceFactory;
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.core.TimerManager;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.option.CopyOptions;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.SnapshotExecutorOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosure;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotExecutorImpl;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotMetaTable;
+import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotStorage;
+import com.alipay.sofa.jraft.storage.snapshot.local.LocalSnapshotWriter;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+import org.mockito.stubbing.Answer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class SnapshotExecutorTest extends BaseStorageTest {
+    private SnapshotExecutorImpl   executor;
+    @Mock
+    private NodeImpl               node;
+    @Mock
+    private FSMCaller              fSMCaller;
+    @Mock
+    private LogManager             logManager;
+    private Endpoint               addr;
+    @Mock
+    private RpcContext             asyncCtx;
+
+    @Mock
+    private RaftClientService      raftClientService;
+    private String                 uri;
+    private final String           hostPort = "localhost:8081";
+    private final int              readerId = 99;
+    private CopyOptions            copyOpts;
+    private LocalSnapshotMetaTable table;
+    private LocalSnapshotWriter    writer;
+    private LocalSnapshotReader    reader;
+    private RaftOptions            raftOptions;
+    @Mock
+    private LocalSnapshotStorage   snapshotStorage;
+    private TimerManager           timerManager;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.timerManager = new TimerManager(5);
+        this.raftOptions = new RaftOptions();
+        this.writer = new LocalSnapshotWriter(this.path, this.snapshotStorage, this.raftOptions);
+        this.reader = new LocalSnapshotReader(this.snapshotStorage, null, new Endpoint("localhost", 8081),
+            this.raftOptions, this.path);
+
+        Mockito.when(this.snapshotStorage.open()).thenReturn(this.reader);
+        Mockito.when(this.snapshotStorage.create(true)).thenReturn(this.writer);
+
+        this.table = new LocalSnapshotMetaTable(this.raftOptions);
+        this.table.addFile("testFile", LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test").build());
+        this.table.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(1).build());
+        this.uri = "remote://" + this.hostPort + "/" + this.readerId;
+        this.copyOpts = new CopyOptions();
+
+        Mockito.when(this.node.getRaftOptions()).thenReturn(new RaftOptions());
+        Mockito.when(this.node.getOptions()).thenReturn(new NodeOptions());
+        Mockito.when(this.node.getRpcService()).thenReturn(this.raftClientService);
+        Mockito.when(this.node.getTimerManager()).thenReturn(this.timerManager);
+        Mockito.when(this.node.getServiceFactory()).thenReturn(DefaultJRaftServiceFactory.newInstance());
+        this.executor = new SnapshotExecutorImpl();
+        final SnapshotExecutorOptions opts = new SnapshotExecutorOptions();
+        opts.setFsmCaller(this.fSMCaller);
+        opts.setInitTerm(0);
+        opts.setNode(this.node);
+        opts.setLogManager(this.logManager);
+        opts.setUri(this.path);
+        this.addr = new Endpoint("localhost", 8081);
+        opts.setAddr(this.addr);
+        assertTrue(this.executor.init(opts));
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        this.executor.shutdown();
+        super.teardown();
+        this.timerManager.shutdown();
+    }
+
+    @Test
+    public void testRetryInstallSnapshot() throws Exception {
+        final RpcRequests.InstallSnapshotRequest.Builder irb = RpcRequests.InstallSnapshotRequest.newBuilder();
+        irb.setGroupId("test");
+        irb.setPeerId(this.addr.toString());
+        irb.setServerId("localhost:8080");
+        irb.setUri("remote://localhost:8080/99");
+        irb.setTerm(0);
+        irb.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(2));
+
+        Mockito.when(this.raftClientService.connect(new Endpoint("localhost", 8080))).thenReturn(true);
+
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+
+        final CountDownLatch retryLatch = new CountDownLatch(1);
+        final CountDownLatch answerLatch = new CountDownLatch(1);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenAnswer(new Answer>() {
+            AtomicInteger count = new AtomicInteger(0);
+
+            @Override
+            public Future answer(InvocationOnMock invocation) throws Throwable {
+                if (count.incrementAndGet() == 1) {
+                    retryLatch.countDown();
+                    answerLatch.await();
+                    Thread.sleep(1000);
+                    return future;
+                } else {
+                    throw new IllegalStateException("shouldn't be called more than once");
+                }
+            }
+        });
+
+        final MockAsyncContext installContext = new MockAsyncContext();
+        final MockAsyncContext retryInstallContext = new MockAsyncContext();
+        Utils.runInThread(new Runnable() {
+            @Override
+            public void run() {
+                SnapshotExecutorTest.this.executor.installSnapshot(irb.build(),
+                    RpcRequests.InstallSnapshotResponse.newBuilder(), new RpcRequestClosure(installContext));
+            }
+        });
+
+        Thread.sleep(500);
+        retryLatch.await();
+        Utils.runInThread(new Runnable() {
+            @Override
+            public void run() {
+                answerLatch.countDown();
+                SnapshotExecutorTest.this.executor.installSnapshot(irb.build(),
+                    RpcRequests.InstallSnapshotResponse.newBuilder(), new RpcRequestClosure(retryInstallContext));
+            }
+        });
+
+        RpcResponseClosure closure = argument.getValue();
+        final ByteBuffer metaBuf = this.table.saveToByteBufferAsRemote();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(metaBuf.remaining()).setEof(true)
+            .setData(ByteString.copyFrom(metaBuf)).build());
+
+        //mock get file
+        argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        rb.setFilename("testFile");
+        rb.setCount(this.raftOptions.getMaxByteCountPerRpc());
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+
+        closure.run(Status.OK());
+        Thread.sleep(500);
+        closure = argument.getValue();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
+            .setData(ByteString.copyFrom(new byte[100])).build());
+
+        final ArgumentCaptor loadSnapshotArg = ArgumentCaptor.forClass(LoadSnapshotClosure.class);
+        Mockito.when(this.fSMCaller.onSnapshotLoad(loadSnapshotArg.capture())).thenReturn(true);
+        closure.run(Status.OK());
+        Thread.sleep(2000);
+        final LoadSnapshotClosure done = loadSnapshotArg.getValue();
+        final SnapshotReader reader = done.start();
+        assertNotNull(reader);
+        assertEquals(1, reader.listFiles().size());
+        assertTrue(reader.listFiles().contains("testFile"));
+        done.run(Status.OK());
+        this.executor.join();
+        assertEquals(2, this.executor.getLastSnapshotTerm());
+        assertEquals(1, this.executor.getLastSnapshotIndex());
+        assertNotNull(installContext.getResponseObject());
+        assertNotNull(retryInstallContext.getResponseObject());
+        assertEquals(installContext.as(RpcRequests.ErrorResponse.class).getErrorCode(), RaftError.EINTR.getNumber());
+        assertTrue(retryInstallContext.as(RpcRequests.InstallSnapshotResponse.class).hasSuccess());
+
+    }
+
+    @Test
+    public void testInstallSnapshot() throws Exception {
+        final RpcRequests.InstallSnapshotRequest.Builder irb = RpcRequests.InstallSnapshotRequest.newBuilder();
+        irb.setGroupId("test");
+        irb.setPeerId(this.addr.toString());
+        irb.setServerId("localhost:8080");
+        irb.setUri("remote://localhost:8080/99");
+        irb.setTerm(0);
+        irb.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(2));
+
+        Mockito.when(this.raftClientService.connect(new Endpoint("localhost", 8080))).thenReturn(true);
+
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                SnapshotExecutorTest.this.executor.installSnapshot(irb.build(), RpcRequests.InstallSnapshotResponse
+                    .newBuilder(), new RpcRequestClosure(SnapshotExecutorTest.this.asyncCtx));
+
+            }
+        });
+
+        Thread.sleep(500);
+        RpcResponseClosure closure = argument.getValue();
+        final ByteBuffer metaBuf = this.table.saveToByteBufferAsRemote();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(metaBuf.remaining()).setEof(true)
+            .setData(ByteString.copyFrom(metaBuf)).build());
+
+        //mock get file
+        argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        rb.setFilename("testFile");
+        rb.setCount(this.raftOptions.getMaxByteCountPerRpc());
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+
+        closure.run(Status.OK());
+
+        Thread.sleep(500);
+        closure = argument.getValue();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
+            .setData(ByteString.copyFrom(new byte[100])).build());
+
+        final ArgumentCaptor loadSnapshotArg = ArgumentCaptor.forClass(LoadSnapshotClosure.class);
+        Mockito.when(this.fSMCaller.onSnapshotLoad(loadSnapshotArg.capture())).thenReturn(true);
+        closure.run(Status.OK());
+        Thread.sleep(500);
+        final LoadSnapshotClosure done = loadSnapshotArg.getValue();
+        final SnapshotReader reader = done.start();
+        assertNotNull(reader);
+        assertEquals(1, reader.listFiles().size());
+        assertTrue(reader.listFiles().contains("testFile"));
+        done.run(Status.OK());
+        this.executor.join();
+        assertEquals(2, this.executor.getLastSnapshotTerm());
+        assertEquals(1, this.executor.getLastSnapshotIndex());
+    }
+
+    @Test
+    public void testInterruptInstallaling() throws Exception {
+        final RpcRequests.InstallSnapshotRequest.Builder irb = RpcRequests.InstallSnapshotRequest.newBuilder();
+        irb.setGroupId("test");
+        irb.setPeerId(this.addr.toString());
+        irb.setServerId("localhost:8080");
+        irb.setUri("remote://localhost:8080/99");
+        irb.setTerm(0);
+        irb.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(1));
+
+        Mockito.when(this.raftClientService.connect(new Endpoint("localhost", 8080))).thenReturn(true);
+
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        final ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8080)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                SnapshotExecutorTest.this.executor.installSnapshot(irb.build(), RpcRequests.InstallSnapshotResponse
+                    .newBuilder(), new RpcRequestClosure(SnapshotExecutorTest.this.asyncCtx));
+
+            }
+        });
+
+        this.executor.interruptDownloadingSnapshots(1);
+        this.executor.join();
+        assertEquals(0, this.executor.getLastSnapshotTerm());
+        assertEquals(0, this.executor.getLastSnapshotIndex());
+    }
+
+    @Test
+    public void testDoSnapshot() throws Exception {
+        Mockito.when(this.fSMCaller.getLastAppliedIndex()).thenReturn(1L);
+        final ArgumentCaptor saveSnapshotClosureArg = ArgumentCaptor
+            .forClass(SaveSnapshotClosure.class);
+        Mockito.when(this.fSMCaller.onSnapshotSave(saveSnapshotClosureArg.capture())).thenReturn(true);
+        final SynchronizedClosure done = new SynchronizedClosure();
+        this.executor.doSnapshot(done);
+        final SaveSnapshotClosure closure = saveSnapshotClosureArg.getValue();
+        assertNotNull(closure);
+        closure.start(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(2).setLastIncludedTerm(1).build());
+        closure.run(Status.OK());
+        done.await();
+        this.executor.join();
+        assertTrue(done.getStatus().isOk());
+        assertEquals(1, this.executor.getLastSnapshotTerm());
+        assertEquals(2, this.executor.getLastSnapshotIndex());
+    }
+
+    @Test
+    public void testNotDoSnapshotWithIntervalDist() throws Exception {
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setSnapshotLogIndexMargin(10);
+        Mockito.when(this.node.getOptions()).thenReturn(nodeOptions);
+        Mockito.when(this.fSMCaller.getLastAppliedIndex()).thenReturn(1L);
+        this.executor.doSnapshot(null);
+        this.executor.join();
+
+        assertEquals(0, this.executor.getLastSnapshotTerm());
+        assertEquals(0, this.executor.getLastSnapshotIndex());
+
+    }
+
+    @Test
+    public void testDoSnapshotWithIntervalDist() throws Exception {
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setSnapshotLogIndexMargin(5);
+        Mockito.when(this.node.getOptions()).thenReturn(nodeOptions);
+        Mockito.when(this.fSMCaller.getLastAppliedIndex()).thenReturn(6L);
+
+        final ArgumentCaptor saveSnapshotClosureArg = ArgumentCaptor
+            .forClass(SaveSnapshotClosure.class);
+        Mockito.when(this.fSMCaller.onSnapshotSave(saveSnapshotClosureArg.capture())).thenReturn(true);
+        final SynchronizedClosure done = new SynchronizedClosure();
+        this.executor.doSnapshot(done);
+        final SaveSnapshotClosure closure = saveSnapshotClosureArg.getValue();
+        assertNotNull(closure);
+        closure.start(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(6).setLastIncludedTerm(1).build());
+        closure.run(Status.OK());
+        done.await();
+        this.executor.join();
+
+        assertEquals(1, this.executor.getLastSnapshotTerm());
+        assertEquals(6, this.executor.getLastSnapshotIndex());
+
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/BaseLogStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/BaseLogStorageTest.java
new file mode 100644
index 0000000..b13cca7
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/BaseLogStorageTest.java
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.conf.ConfigurationManager;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.codec.LogEntryCodecFactory;
+import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory;
+import com.alipay.sofa.jraft.option.LogStorageOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Utils;
+
+public abstract class BaseLogStorageTest extends BaseStorageTest {
+    protected LogStorage         logStorage;
+    private ConfigurationManager confManager;
+    private LogEntryCodecFactory logEntryCodecFactory;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.confManager = new ConfigurationManager();
+        this.logEntryCodecFactory = LogEntryV2CodecFactory.getInstance();
+        this.logStorage = newLogStorage();
+
+        final LogStorageOptions opts = newLogStorageOptions();
+
+        this.logStorage.init(opts);
+    }
+
+    protected abstract LogStorage newLogStorage();
+
+    protected LogStorageOptions newLogStorageOptions() {
+        final LogStorageOptions opts = new LogStorageOptions();
+        opts.setConfigurationManager(this.confManager);
+        opts.setLogEntryCodecFactory(this.logEntryCodecFactory);
+        return opts;
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        this.logStorage.shutdown();
+        super.teardown();
+    }
+
+    @Test
+    public void testEmptyState() {
+        assertEquals(1, this.logStorage.getFirstLogIndex());
+        assertEquals(0, this.logStorage.getLastLogIndex());
+        assertNull(this.logStorage.getEntry(100));
+        assertEquals(0, this.logStorage.getTerm(100));
+    }
+
+    @Test
+    public void testAddOneEntryState() {
+        final LogEntry entry1 = TestUtils.mockEntry(100, 1);
+        assertTrue(this.logStorage.appendEntry(entry1));
+
+        assertEquals(100, this.logStorage.getFirstLogIndex());
+        assertEquals(100, this.logStorage.getLastLogIndex());
+        Assert.assertEquals(entry1, this.logStorage.getEntry(100));
+        assertEquals(1, this.logStorage.getTerm(100));
+
+        final LogEntry entry2 = TestUtils.mockEntry(200, 2);
+        assertTrue(this.logStorage.appendEntry(entry2));
+
+        assertEquals(100, this.logStorage.getFirstLogIndex());
+        assertEquals(200, this.logStorage.getLastLogIndex());
+        Assert.assertEquals(entry1, this.logStorage.getEntry(100));
+        Assert.assertEquals(entry2, this.logStorage.getEntry(200));
+        assertEquals(1, this.logStorage.getTerm(100));
+        assertEquals(2, this.logStorage.getTerm(200));
+    }
+
+    @Test
+    public void testLoadWithConfigManager() {
+        assertTrue(this.confManager.getLastConfiguration().isEmpty());
+
+        final LogEntry confEntry1 = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION);
+        confEntry1.setId(new LogId(99, 1));
+        confEntry1.setPeers(JRaftUtils.getConfiguration("localhost:8081,localhost:8082").listPeers());
+
+        final LogEntry confEntry2 = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION);
+        confEntry2.setId(new LogId(100, 2));
+        confEntry2.setPeers(JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083").listPeers());
+
+        assertTrue(this.logStorage.appendEntry(confEntry1));
+        assertEquals(1, this.logStorage.appendEntries(Arrays.asList(confEntry2)));
+
+        // reload log storage.
+        this.logStorage.shutdown();
+        this.logStorage = newLogStorage();
+        this.logStorage.init(newLogStorageOptions());
+
+        ConfigurationEntry conf = this.confManager.getLastConfiguration();
+        assertNotNull(conf);
+        assertFalse(conf.isEmpty());
+        assertEquals("localhost:8081,localhost:8082,localhost:8083", conf.getConf().toString());
+        conf = this.confManager.get(99);
+        assertNotNull(conf);
+        assertFalse(conf.isEmpty());
+        assertEquals("localhost:8081,localhost:8082", conf.getConf().toString());
+    }
+
+    @Test
+    public void testAddManyEntries() {
+        final List entries = TestUtils.mockEntries();
+
+        assertEquals(10, this.logStorage.appendEntries(entries));
+
+        assertEquals(0, this.logStorage.getFirstLogIndex());
+        assertEquals(9, this.logStorage.getLastLogIndex());
+        for (int i = 0; i < 10; i++) {
+            assertEquals(i, this.logStorage.getTerm(i));
+            final LogEntry entry = this.logStorage.getEntry(i);
+            assertNotNull(entry);
+            assertEquals(entries.get(i), entry);
+        }
+    }
+
+    @Test
+    public void testReset() {
+        testAddManyEntries();
+        this.logStorage.reset(5);
+        assertEquals(5, this.logStorage.getFirstLogIndex());
+        assertEquals(5, this.logStorage.getLastLogIndex());
+        assertEquals(5, this.logStorage.getTerm(5));
+    }
+
+    @Test
+    public void testTruncatePrefix() throws Exception {
+        final List entries = TestUtils.mockEntries();
+
+        assertEquals(10, this.logStorage.appendEntries(entries));
+        this.logStorage.truncatePrefix(5);
+        Thread.sleep(1000);
+        assertEquals(5, this.logStorage.getFirstLogIndex());
+        assertEquals(9, this.logStorage.getLastLogIndex());
+        for (int i = 0; i < 10; i++) {
+            if (i < 5) {
+                assertNull(this.logStorage.getEntry(i));
+                if (this.logStorage instanceof RocksDBLogStorage) {
+                    assertNull(((RocksDBLogStorage) this.logStorage).getEntryFromDB(i));
+                }
+            } else {
+                Assert.assertEquals(entries.get(i), this.logStorage.getEntry(i));
+            }
+        }
+    }
+
+    @Test
+    public void testAppendMantyLargeEntries() {
+        final long start = Utils.monotonicMs();
+        final int totalLogs = 100000;
+        final int logSize = 16 * 1024;
+        final int batch = 100;
+
+        appendLargeEntries(totalLogs, logSize, batch);
+
+        System.out.println("Inserted " + totalLogs + " large logs, cost " + (Utils.monotonicMs() - start) + " ms.");
+
+        for (int i = 0; i < totalLogs; i++) {
+            final LogEntry log = this.logStorage.getEntry(i);
+            assertNotNull(log);
+            assertEquals(i, log.getId().getIndex());
+            assertEquals(i, log.getId().getTerm());
+            assertEquals(logSize, log.getData().remaining());
+        }
+
+        this.logStorage.shutdown();
+        this.logStorage.init(newLogStorageOptions());
+
+        for (int i = 0; i < totalLogs; i++) {
+            final LogEntry log = this.logStorage.getEntry(i);
+            assertNotNull(log);
+            assertEquals(i, log.getId().getIndex());
+            assertEquals(i, log.getId().getTerm());
+            assertEquals(logSize, log.getData().remaining());
+        }
+    }
+
+    private void appendLargeEntries(final int totalLogs, final int logSize, final int batch) {
+        for (int i = 0; i < totalLogs; i += batch) {
+            final List entries = new ArrayList<>(batch);
+            for (int j = i; j < i + batch; j++) {
+                entries.add(TestUtils.mockEntry(j, j, logSize));
+            }
+            assertEquals(batch, this.logStorage.appendEntries(entries));
+        }
+    }
+
+    @Test
+    public void testTruncateSuffix() {
+        final List entries = TestUtils.mockEntries();
+
+        assertEquals(10, this.logStorage.appendEntries(entries));
+        this.logStorage.truncateSuffix(5);
+        assertEquals(0, this.logStorage.getFirstLogIndex());
+        assertEquals(5, this.logStorage.getLastLogIndex());
+        for (int i = 0; i < 10; i++) {
+            if (i <= 5) {
+                Assert.assertEquals(entries.get(i), this.logStorage.getEntry(i));
+            } else {
+                assertNull(this.logStorage.getEntry(i));
+            }
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorageTest.java
new file mode 100644
index 0000000..fe52645
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LocalRaftMetaStorageTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.option.RaftMetaStorageOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.RaftMetaStorage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-11 4:50:23 PM
+ */
+@RunWith(MockitoJUnitRunner.class)
+public class LocalRaftMetaStorageTest extends BaseStorageTest {
+    private RaftMetaStorage raftMetaStorage;
+
+    @Mock
+    private NodeImpl        node;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.raftMetaStorage = new LocalRaftMetaStorage(this.path, new RaftOptions());
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(null);
+        assertTrue(this.raftMetaStorage.init(newOptions()));
+    }
+
+    private RaftMetaStorageOptions newOptions() {
+        RaftMetaStorageOptions raftMetaStorageOptions = new RaftMetaStorageOptions();
+        raftMetaStorageOptions.setNode(this.node);
+        return raftMetaStorageOptions;
+    }
+
+    @Test
+    public void testGetAndSetReload() {
+        assertEquals(0, this.raftMetaStorage.getTerm());
+        assertTrue(this.raftMetaStorage.getVotedFor().isEmpty());
+
+        this.raftMetaStorage.setTerm(99);
+        assertEquals(99, this.raftMetaStorage.getTerm());
+        assertTrue(this.raftMetaStorage.getVotedFor().isEmpty());
+
+        assertTrue(this.raftMetaStorage.setVotedFor(new PeerId("localhost", 8081)));
+        assertEquals(99, this.raftMetaStorage.getTerm());
+        Assert.assertEquals(new PeerId("localhost", 8081), this.raftMetaStorage.getVotedFor());
+
+        assertTrue(this.raftMetaStorage.setTermAndVotedFor(100, new PeerId("localhost", 8083)));
+        assertEquals(100, this.raftMetaStorage.getTerm());
+        Assert.assertEquals(new PeerId("localhost", 8083), this.raftMetaStorage.getVotedFor());
+
+        this.raftMetaStorage = new LocalRaftMetaStorage(this.path, new RaftOptions());
+        Mockito.when(this.node.getNodeMetrics()).thenReturn(null);
+        this.raftMetaStorage.init(newOptions());
+        assertEquals(100, this.raftMetaStorage.getTerm());
+        Assert.assertEquals(new PeerId("localhost", 8083), this.raftMetaStorage.getVotedFor());
+    }
+
+    @Test
+    public void testSaveFail() throws IOException {
+        FileUtils.deleteDirectory(new File(this.path));
+        assertFalse(this.raftMetaStorage.setVotedFor(new PeerId("localhost", 8081)));
+        Mockito.verify(this.node, Mockito.times(1)).onError((RaftException) Mockito.any());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerTest.java
new file mode 100644
index 0000000..ecfe2b7
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerTest.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.FSMCaller;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.conf.ConfigurationManager;
+import com.alipay.sofa.jraft.core.NodeMetrics;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory;
+import com.alipay.sofa.jraft.option.LogManagerOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.LogManager;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class LogManagerTest extends BaseStorageTest {
+    private LogManagerImpl       logManager;
+    private ConfigurationManager confManager;
+    @Mock
+    private FSMCaller            fsmCaller;
+
+    private LogStorage           logStorage;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.confManager = new ConfigurationManager();
+        final RaftOptions raftOptions = new RaftOptions();
+        this.logStorage = newLogStorage(raftOptions);
+        this.logManager = new LogManagerImpl();
+        final LogManagerOptions opts = new LogManagerOptions();
+        opts.setConfigurationManager(this.confManager);
+        opts.setLogEntryCodecFactory(LogEntryV2CodecFactory.getInstance());
+        opts.setFsmCaller(this.fsmCaller);
+        opts.setNodeMetrics(new NodeMetrics(false));
+        opts.setLogStorage(this.logStorage);
+        opts.setRaftOptions(raftOptions);
+        assertTrue(this.logManager.init(opts));
+    }
+
+    protected RocksDBLogStorage newLogStorage(final RaftOptions raftOptions) {
+        return new RocksDBLogStorage(this.path, raftOptions);
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        this.logStorage.shutdown();
+        super.teardown();
+    }
+
+    @Test
+    public void testEmptyState() {
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(0, this.logManager.getLastLogIndex());
+        assertNull(this.logManager.getEntry(1));
+        assertEquals(0, this.logManager.getLastLogIndex(true));
+        LogId lastLogId = this.logManager.getLastLogId(true);
+        assertEquals(0, lastLogId.getIndex());
+        lastLogId = this.logManager.getLastLogId(false);
+        assertEquals(0, lastLogId.getIndex());
+        assertTrue(this.logManager.checkConsistency().isOk());
+    }
+
+    @Test
+    public void testHasAvailableCapacityToAppendEntries() {
+        assertTrue(this.logManager.hasAvailableCapacityToAppendEntries(1));
+        assertTrue(this.logManager.hasAvailableCapacityToAppendEntries(10));
+        assertFalse(this.logManager.hasAvailableCapacityToAppendEntries(1000000));
+    }
+
+    @Test
+    public void testAppendOneEntry() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final LogEntry entry = TestUtils.mockEntry(1, 1);
+        final List entries = new ArrayList<>();
+        entries.add(entry);
+        this.logManager.appendEntries(entries, new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+        latch.await();
+
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(1, this.logManager.getLastLogIndex());
+        Assert.assertEquals(entry, this.logManager.getEntry(1));
+        assertEquals(1, this.logManager.getLastLogIndex(true));
+        LogId lastLogId = this.logManager.getLastLogId(true);
+        assertEquals(1, lastLogId.getIndex());
+        lastLogId = this.logManager.getLastLogId(false);
+        assertEquals(1, lastLogId.getIndex());
+        assertTrue(this.logManager.checkConsistency().isOk());
+    }
+
+    @Test
+    public void testAppendEntries() throws Exception {
+        final List mockEntries = mockAddEntries();
+
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(10, this.logManager.getLastLogIndex());
+        for (int i = 0; i < 10; i++) {
+            Assert.assertEquals(mockEntries.get(i), this.logManager.getEntry(i + 1));
+        }
+        assertEquals(10, this.logManager.getLastLogIndex(true));
+        LogId lastLogId = this.logManager.getLastLogId(true);
+        assertEquals(10, lastLogId.getIndex());
+        lastLogId = this.logManager.getLastLogId(false);
+        assertEquals(10, lastLogId.getIndex());
+        assertTrue(this.logManager.checkConsistency().isOk());
+    }
+
+    @Test
+    public void testAppendEntriesBeforeAppliedIndex() throws Exception {
+        //Append 0-10
+        List mockEntries = TestUtils.mockEntries(10);
+        for (int i = 0; i < 10; i++) {
+            mockEntries.get(i).getId().setTerm(1);
+        }
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch1.countDown();
+            }
+        });
+        latch1.await();
+
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(10, this.logManager.getLastLogIndex());
+        Thread.sleep(200); // waiting for setDiskId()
+        this.logManager.setAppliedId(new LogId(9, 1));
+
+        for (int i = 0; i < 10; i++) {
+            assertNull(this.logManager.getEntryFromMemory(i));
+        }
+
+        // append 1-10 again, already applied, returns OK.
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        mockEntries = TestUtils.mockEntries(10);
+        mockEntries.remove(0);
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch2.countDown();
+            }
+        });
+        latch2.await();
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(10, this.logManager.getLastLogIndex());
+    }
+
+    @Test
+    public void testAppendEntresConflicts() throws Exception {
+        //Append 0-10
+        List mockEntries = TestUtils.mockEntries(10);
+        for (int i = 0; i < 10; i++) {
+            mockEntries.get(i).getId().setTerm(1);
+        }
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch1.countDown();
+            }
+        });
+        latch1.await();
+
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(10, this.logManager.getLastLogIndex());
+
+        //Append 11-20
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        mockEntries = TestUtils.mockEntries(10);
+        for (int i = 0; i < 10; i++) {
+            mockEntries.get(i).getId().setIndex(11 + i);
+            mockEntries.get(i).getId().setTerm(1);
+        }
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch2.countDown();
+            }
+        });
+        latch2.await();
+
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(20, this.logManager.getLastLogIndex());
+
+        //Re-adds 11-30, but 15 has different term, it will truncate [14,lastIndex] logs
+        mockEntries = TestUtils.mockEntries(20);
+        for (int i = 0; i < 20; i++) {
+            if (11 + i >= 15) {
+                mockEntries.get(i).getId().setTerm(2);
+            } else {
+                mockEntries.get(i).getId().setTerm(1);
+            }
+            mockEntries.get(i).getId().setIndex(11 + i);
+        }
+        final CountDownLatch latch3 = new CountDownLatch(1);
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch3.countDown();
+            }
+        });
+        latch3.await();
+        assertEquals(1, this.logManager.getFirstLogIndex());
+        assertEquals(30, this.logManager.getLastLogIndex());
+
+        for (int i = 0; i < 30; i++) {
+            final LogEntry entry = (this.logManager.getEntry(i + 1));
+            assertEquals(i + 1, entry.getId().getIndex());
+            if (i + 1 >= 15) {
+                assertEquals(2, entry.getId().getTerm());
+            } else {
+                assertEquals(1, entry.getId().getTerm());
+            }
+        }
+    }
+
+    @Test
+    public void testGetConfiguration() throws Exception {
+        assertTrue(this.logManager.getConfiguration(1).isEmpty());
+        final List entries = new ArrayList<>(2);
+        final LogEntry confEntry1 = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION);
+        confEntry1.setId(new LogId(0, 1));
+        confEntry1.setPeers(JRaftUtils.getConfiguration("localhost:8081,localhost:8082").listPeers());
+
+        final LogEntry confEntry2 = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_CONFIGURATION);
+        confEntry2.setId(new LogId(0, 2));
+        confEntry2.setPeers(JRaftUtils.getConfiguration("localhost:8081,localhost:8082,localhost:8083").listPeers());
+        confEntry2.setOldPeers(confEntry1.getPeers());
+
+        entries.add(confEntry1);
+        entries.add(confEntry2);
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        this.logManager.appendEntries(new ArrayList<>(entries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+        latch.await();
+        ConfigurationEntry entry = this.logManager.getConfiguration(1);
+        assertEquals("localhost:8081,localhost:8082", entry.getConf().toString());
+        assertTrue(entry.getOldConf().isEmpty());
+
+        entry = this.logManager.getConfiguration(2);
+        assertEquals("localhost:8081,localhost:8082,localhost:8083", entry.getConf().toString());
+        assertEquals("localhost:8081,localhost:8082", entry.getOldConf().toString());
+    }
+
+    @Test
+    public void testSetAppliedId() throws Exception {
+        final List mockEntries = mockAddEntries();
+
+        for (int i = 0; i < 10; i++) {
+            // it's in memory
+            Assert.assertEquals(mockEntries.get(i), this.logManager.getEntryFromMemory(i + 1));
+        }
+        Thread.sleep(200); // waiting for setDiskId()
+        this.logManager.setAppliedId(new LogId(10, 10));
+        for (int i = 0; i < 10; i++) {
+            assertNull(this.logManager.getEntryFromMemory(i + 1));
+            Assert.assertEquals(mockEntries.get(i), this.logManager.getEntry(i + 1));
+        }
+    }
+
+    @Test
+    public void testSetAppliedId2() throws Exception {
+        final List mockEntries = mockAddEntries();
+
+        for (int i = 0; i < 10; i++) {
+            // it's in memory
+            Assert.assertEquals(mockEntries.get(i), this.logManager.getEntryFromMemory(i + 1));
+        }
+        Thread.sleep(200); // waiting for setDiskId()
+        this.logManager.setAppliedId(new LogId(10, 10));
+        for (int i = 0; i < 10; i++) {
+            assertNull(this.logManager.getEntryFromMemory(i + 1));
+            Assert.assertEquals(mockEntries.get(i), this.logManager.getEntry(i + 1));
+        }
+    }
+
+    private List mockAddEntries() throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(1);
+        final List mockEntries = TestUtils.mockEntries(10);
+        this.logManager.appendEntries(new ArrayList<>(mockEntries), new LogManager.StableClosure() {
+
+            @Override
+            public void run(final Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+        latch.await();
+        return mockEntries;
+    }
+
+    @Test
+    public void testSetSnapshot() throws Exception {
+        final List entries = mockAddEntries();
+        RaftOutter.SnapshotMeta meta = RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(3)
+            .setLastIncludedTerm(2).addPeers("localhost:8081").build();
+        this.logManager.setSnapshot(meta);
+        //Still valid
+        for (int i = 0; i < 10; i++) {
+            Assert.assertEquals(entries.get(i), this.logManager.getEntry(i + 1));
+        }
+        meta = RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(5).setLastIncludedTerm(4)
+            .addPeers("localhost:8081").build();
+        this.logManager.setSnapshot(meta);
+
+        Thread.sleep(1000);
+        for (int i = 0; i < 10; i++) {
+            if (i > 2) {
+                Assert.assertEquals(entries.get(i), this.logManager.getEntry(i + 1));
+            } else {
+                //before index=3 logs were dropped.
+                assertNull(this.logManager.getEntry(i + 1));
+            }
+        }
+        assertTrue(this.logManager.checkConsistency().isOk());
+    }
+
+    @Test
+    public void testWaiter() throws Exception {
+        mockAddEntries();
+        final Object theArg = new Object();
+        final CountDownLatch latch = new CountDownLatch(1);
+        final long waitId = this.logManager.wait(10, (arg, errorCode) -> {
+            assertSame(arg, theArg);
+            assertEquals(0, errorCode);
+            latch.countDown();
+            return true;
+        }, theArg);
+        assertEquals(1, waitId);
+        mockAddEntries();
+        latch.await();
+        assertFalse(this.logManager.removeWaiter(waitId));
+    }
+
+    @Test
+    public void testCheckAndSetConfiguration() throws Exception {
+        assertNull(this.logManager.checkAndSetConfiguration(null));
+        final ConfigurationEntry entry = new ConfigurationEntry();
+        entry.setId(new LogId(0, 1));
+        entry.setConf(JRaftUtils.getConfiguration("localhost:8081,localhost:8082"));
+        assertSame(entry, this.logManager.checkAndSetConfiguration(entry));
+
+        testGetConfiguration();
+        final ConfigurationEntry lastEntry = this.logManager.checkAndSetConfiguration(entry);
+        assertNotSame(entry, lastEntry);
+        assertEquals("localhost:8081,localhost:8082,localhost:8083", lastEntry.getConf().toString());
+        assertEquals("localhost:8081,localhost:8082", lastEntry.getOldConf().toString());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerWithSegmentLogStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerWithSegmentLogStorageTest.java
new file mode 100644
index 0000000..d4069e2
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogManagerWithSegmentLogStorageTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage;
+
+public class LogManagerWithSegmentLogStorageTest extends LogManagerTest {
+
+    @Override
+    protected RocksDBLogStorage newLogStorage(final RaftOptions raftOptions) {
+        return new RocksDBSegmentLogStorage(this.path, raftOptions, 0, 64 * 1024);
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogStorageBenchmark.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogStorageBenchmark.java
new file mode 100644
index 0000000..713a564
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/LogStorageBenchmark.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alipay.sofa.jraft.conf.ConfigurationManager;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.codec.v2.LogEntryV2CodecFactory;
+import com.alipay.sofa.jraft.option.LogStorageOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.alipay.sofa.jraft.util.Utils;
+
+public class LogStorageBenchmark {
+
+    private final LogStorage logStorage;
+
+    private final int        logSize;
+
+    private final int        totalLogs;
+
+    private final int        batchSize;
+
+    public LogStorageBenchmark(final LogStorage logStorage, final int logSize, final int totalLogs, final int batchSize) {
+        super();
+        this.logStorage = logStorage;
+        this.logSize = logSize;
+        this.totalLogs = totalLogs;
+        this.batchSize = batchSize;
+    }
+
+    private void write(final int batchSize, final int logSize, final int totalLogs) {
+        List entries = new ArrayList<>(batchSize);
+        for (int i = 0; i < totalLogs; i += batchSize) {
+            for (int j = i; j < i + batchSize; j++) {
+                entries.add(TestUtils.mockEntry(j, j, logSize));
+            }
+            int ret = this.logStorage.appendEntries(entries);
+            if (ret != batchSize) {
+                System.err.println("Fatal error: write failures, expect " + batchSize + ", but was " + ret);
+                System.exit(1);
+            }
+            entries.clear(); //reuse it
+        }
+    }
+
+    private static void assertNotNull(final Object obj) {
+        if (obj == null) {
+            System.err.println("Null object");
+            System.exit(1);
+        }
+    }
+
+    private static void assertEquals(final long x, final long y) {
+        if (x != y) {
+            System.err.println("Expect " + x + " but was " + y);
+            System.exit(1);
+        }
+    }
+
+    private void read(final int logSize, final int totalLogs) {
+        for (int i = 0; i < totalLogs; i++) {
+            LogEntry log = this.logStorage.getEntry(i);
+            assertNotNull(log);
+            assertEquals(i, log.getId().getIndex());
+            assertEquals(i, log.getId().getTerm());
+            assertEquals(logSize, log.getData().remaining());
+        }
+    }
+
+    private void report(final String op, final long cost) {
+        System.out.println("Test " + op + ":");
+        System.out.println("  Log number   :" + this.totalLogs);
+        System.out.println("  Log Size     :" + this.logSize);
+        System.out.println("  Batch Size   :" + this.batchSize);
+        System.out.println("  Cost time(s) :" + cost / 1000);
+        System.out.println("  Total size   :" + (long) this.totalLogs * this.logSize);
+    }
+
+    private void doTest() {
+        System.out.println("Begin test...");
+        {
+            System.out.println("Warm up...");
+            write(10, 64, 10000);
+            read(64, 10000);
+        }
+
+        System.out.println("Start test...");
+        {
+            long start = Utils.monotonicMs();
+            write(this.batchSize, this.logSize, this.totalLogs);
+            long cost = Utils.monotonicMs() - start;
+            report("write", cost);
+        }
+
+        {
+            long start = Utils.monotonicMs();
+            read(this.logSize, this.totalLogs);
+            long cost = Utils.monotonicMs() - start;
+            report("read", cost);
+        }
+        System.out.println("Test done!");
+    }
+
+    public static void main(final String[] args) {
+        String testPath = Paths.get(SystemPropertyUtil.get("user.dir"), "log_storage").toString();
+        System.out.println("Test log storage path: " + testPath);
+        int batchSize = 100;
+        int logSize = 16 * 1024;
+        int totalLogs = 1024 * 1024;
+
+        // LogStorage logStorage = new RocksDBLogStorage(testPath, new RaftOptions());
+        LogStorage logStorage = new RocksDBSegmentLogStorage(testPath, new RaftOptions());
+
+        LogStorageOptions opts = new LogStorageOptions();
+        opts.setConfigurationManager(new ConfigurationManager());
+        opts.setLogEntryCodecFactory(LogEntryV2CodecFactory.getInstance());
+        logStorage.init(opts);
+
+        new LogStorageBenchmark(logStorage, logSize, totalLogs, batchSize).doTest();
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorageTest.java
new file mode 100644
index 0000000..ceb2436
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBLogStorageTest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.LogStorage;
+
+public class RocksDBLogStorageTest extends BaseLogStorageTest {
+
+    @Override
+    protected LogStorage newLogStorage() {
+        return new RocksDBLogStorage(this.path, new RaftOptions());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBSegmentLogStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBSegmentLogStorageTest.java
new file mode 100644
index 0000000..818a132
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/impl/RocksDBSegmentLogStorageTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.impl;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import java.io.File;
+import java.util.Arrays;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+import com.alipay.sofa.jraft.option.LogStorageOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.LogStorage;
+import com.alipay.sofa.jraft.storage.log.RocksDBSegmentLogStorage;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+public class RocksDBSegmentLogStorageTest extends BaseLogStorageTest {
+
+    @Override
+    protected LogStorage newLogStorage() {
+        return new RocksDBSegmentLogStorage(this.path, new RaftOptions(), 0, 1024 * 1024);
+    }
+
+    @Test
+    public void testTruncateChaos() throws Exception {
+        int times = 100;
+        int n = 100;
+
+        for (int i = 0; i < times; i++) {
+            this.logStorage.shutdown();
+            FileUtils.deleteDirectory(new File(this.path));
+            this.path = TestUtils.mkTempDir();
+            FileUtils.forceMkdir(new File(this.path));
+            this.logStorage = new RocksDBSegmentLogStorage(this.path, new RaftOptions(), 32, 256);
+            final LogStorageOptions opts = newLogStorageOptions();
+            this.logStorage.init(opts);
+
+            for (int j = 0; j < n; j++) {
+                this.logStorage.appendEntries(Arrays.asList(TestUtils.mockEntry(j, 1, ThreadLocalRandom.current()
+                    .nextInt(180))));
+            }
+
+            int index = ThreadLocalRandom.current().nextInt(n);
+            boolean truncatePrefix = ThreadLocalRandom.current().nextBoolean();
+            if (truncatePrefix) {
+                this.logStorage.truncatePrefix(index);
+
+                for (int j = 0; j < n; j++) {
+                    if (j < index) {
+                        assertNull(this.logStorage.getEntry(j));
+                    } else {
+                        assertNotNull(this.logStorage.getEntry(j));
+                    }
+                }
+            } else {
+                this.logStorage.truncateSuffix(index);
+
+                for (int j = 0; j < n; j++) {
+                    if (j <= index) {
+                        assertNotNull(this.logStorage.getEntry(j));
+                    } else {
+                        assertNull(this.logStorage.getEntry(j));
+                    }
+                }
+            }
+        }
+    }
+
+    @Test
+    public void testTruncateSuffixWithDifferentValueSize() throws Exception {
+        // shutdown the old one
+        this.logStorage.shutdown();
+        // Set value threshold to be 32 bytes.
+        this.logStorage = new RocksDBSegmentLogStorage(this.path, new RaftOptions(), 32, 64);
+        final LogStorageOptions opts = newLogStorageOptions();
+        this.logStorage.init(opts);
+        int term = 1;
+
+        for (int i = 0; i < 10; i++) {
+            this.logStorage.appendEntries(Arrays.asList(TestUtils.mockEntry(i, term, i)));
+        }
+
+        this.logStorage.appendEntries(Arrays.asList(TestUtils.mockEntry(10, term, 64)));
+
+        for (int i = 11; i < 20; i++) {
+            this.logStorage.appendEntries(Arrays.asList(TestUtils.mockEntry(i, term, i)));
+        }
+
+        for (int i = 0; i < 20; i++) {
+            assertNotNull(this.logStorage.getEntry(i));
+        }
+
+        assertEquals(((RocksDBSegmentLogStorage) this.logStorage).getLastSegmentFileForRead().getWrotePos(), 179);
+
+        this.logStorage.truncateSuffix(15);
+
+        for (int i = 0; i < 20; i++) {
+            if (i <= 15) {
+                assertNotNull(this.logStorage.getEntry(i));
+            } else {
+                assertNull(this.logStorage.getEntry(i));
+            }
+        }
+
+        assertEquals(((RocksDBSegmentLogStorage) this.logStorage).getLastSegmentFileForRead().getWrotePos(), 102);
+
+        this.logStorage.truncateSuffix(13);
+
+        for (int i = 0; i < 20; i++) {
+            if (i <= 13) {
+                assertNotNull(this.logStorage.getEntry(i));
+            } else {
+                assertNull(this.logStorage.getEntry(i));
+            }
+        }
+
+        assertEquals(((RocksDBSegmentLogStorage) this.logStorage).getLastSegmentFileForRead().getWrotePos(), 102);
+
+        this.logStorage.truncateSuffix(5);
+        for (int i = 0; i < 20; i++) {
+            if (i <= 5) {
+                assertNotNull(this.logStorage.getEntry(i));
+            } else {
+                assertNull(this.logStorage.getEntry(i));
+            }
+        }
+
+        assertNull(((RocksDBSegmentLogStorage) this.logStorage).getLastSegmentFileForRead());
+        this.logStorage.appendEntries(Arrays.asList(TestUtils.mockEntry(20, term, 10)));
+        assertNotNull(this.logStorage.getEntry(20));
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/LocalFileReaderTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/LocalFileReaderTest.java
new file mode 100644
index 0000000..d78452a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/LocalFileReaderTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.io;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+
+public class LocalFileReaderTest extends BaseStorageTest {
+    private LocalDirReader fileReader;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.fileReader = new LocalDirReader(path);
+    }
+
+    @Test
+    public void testReadFile() throws Exception {
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate();
+        try {
+            this.fileReader.readFile(bufRef, "unfound", 0, 1024);
+            fail();
+        } catch (final FileNotFoundException e) {
+
+        }
+
+        final String data = writeData();
+
+        assertReadResult(bufRef, data);
+    }
+
+    private void assertReadResult(ByteBufferCollector bufRef, String data) throws Exception {
+        final int read = this.fileReader.readFile(bufRef, "data", 0, 1024);
+        assertEquals(-1, read);
+        final ByteBuffer buf = bufRef.getBuffer();
+        buf.flip();
+        assertEquals(data.length(), buf.remaining());
+        final byte[] bs = new byte[data.length()];
+        buf.get(bs);
+        assertEquals(data, new String(bs));
+    }
+
+    @Test
+    public void testReadSmallInitBuffer() throws Exception {
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(2);
+
+        final String data = writeData();
+
+        assertReadResult(bufRef, data);
+    }
+
+    @Test
+    public void testReadBigFile() throws Exception {
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(2);
+
+        final File file = new File(this.path + File.separator + "data");
+        String data = "";
+        for (int i = 0; i < 4096; i++) {
+            data += i % 10;
+        }
+        FileUtils.writeStringToFile(file, data);
+
+        int read = this.fileReader.readFile(bufRef, "data", 0, 1024);
+        assertEquals(1024, read);
+        read = this.fileReader.readFile(bufRef, "data", 1024, 1024);
+        assertEquals(1024, read);
+        read = this.fileReader.readFile(bufRef, "data", 1024 + 1024, 1024);
+        assertEquals(1024, read);
+        read = this.fileReader.readFile(bufRef, "data", 1024 + 1024 + 1024, 1024);
+        assertEquals(-1, read);
+
+        final ByteBuffer buf = bufRef.getBuffer();
+        buf.flip();
+        assertEquals(data.length(), buf.remaining());
+        final byte[] bs = new byte[data.length()];
+        buf.get(bs);
+        assertEquals(data, new String(bs));
+
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/ProtobufFileTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/ProtobufFileTest.java
new file mode 100644
index 0000000..091059c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/io/ProtobufFileTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.io;
+
+import java.io.File;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class ProtobufFileTest {
+
+    @Test
+    public void testSaveLoad() throws Exception {
+        File tempFile = File.createTempFile("test", "pfile");
+        String path = tempFile.getAbsolutePath();
+        tempFile.delete();
+        ProtoBufFile file = new ProtoBufFile(path);
+        assertNull(file.load());
+        LocalFileMetaOutter.LocalFileMeta msg = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_REFERENCE).build();
+        assertTrue(file.save(msg, true));
+
+        ProtoBufFile newFile = new ProtoBufFile(path);
+        LocalFileMetaOutter.LocalFileMeta loadedMsg = newFile.load();
+        assertNotNull(loadedMsg);
+        assertEquals("test", loadedMsg.getChecksum());
+        Assert.assertEquals(LocalFileMetaOutter.FileSource.FILE_SOURCE_REFERENCE, loadedMsg.getSource());
+
+        new File(path).delete();
+        assertNull(newFile.load());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/AbortFileTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/AbortFileTest.java
new file mode 100644
index 0000000..0168738
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/AbortFileTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.log;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+
+public class AbortFileTest extends BaseStorageTest {
+    private AbortFile abortFile;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.abortFile = new AbortFile(this.path + File.separator + "abort");
+    }
+
+    @Test
+    public void testMisc() throws Exception {
+        assertFalse(this.abortFile.exists());
+        assertTrue(this.abortFile.create());
+        assertTrue(this.abortFile.exists());
+        assertFalse(this.abortFile.create());
+        this.abortFile.destroy();
+        assertFalse(this.abortFile.exists());
+        assertTrue(this.abortFile.create());
+        assertTrue(this.abortFile.exists());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/CheckpointFileTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/CheckpointFileTest.java
new file mode 100644
index 0000000..e68bb02
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/CheckpointFileTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+
+public class CheckpointFileTest extends BaseStorageTest {
+    private CheckpointFile checkpointFile;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.checkpointFile = new CheckpointFile(this.path + File.separator + "checkpoint");
+    }
+
+    @Test
+    public void testMisc() throws Exception {
+        assertNull(this.checkpointFile.load());
+
+        this.checkpointFile.save(new CheckpointFile.Checkpoint("test1", 99));
+        CheckpointFile.Checkpoint cp = this.checkpointFile.load();
+        assertNotNull(cp);
+        assertEquals("test1", cp.segFilename);
+        assertEquals(99, cp.committedPos);
+
+        this.checkpointFile.destroy();
+        assertNull(this.checkpointFile.load());
+
+        this.checkpointFile.save(new CheckpointFile.Checkpoint("test2", 299));
+        cp = this.checkpointFile.load();
+        assertNotNull(cp);
+        assertEquals("test2", cp.segFilename);
+        assertEquals(299, cp.committedPos);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/SegmentFileTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/SegmentFileTest.java
new file mode 100644
index 0000000..505ae81
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/log/SegmentFileTest.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.log;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.nio.channels.FileChannel;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.remoting.NamedThreadFactory;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.impl.RocksDBLogStorage.WriteContext;
+import com.alipay.sofa.jraft.storage.log.SegmentFile.SegmentFileOptions;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+public class SegmentFileTest extends BaseStorageTest {
+    private static final int   FILE_SIZE = 64 + SegmentFile.HEADER_SIZE;
+    private SegmentFile        segmentFile;
+    private ThreadPoolExecutor writeExecutor;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.writeExecutor = ThreadPoolUtil.newThreadPool("test", false, 10, 10, 60, new SynchronousQueue(),
+            new NamedThreadFactory("test"));
+        String filePath = this.path + File.separator + "SegmentFileTest";
+        this.segmentFile = new SegmentFile(FILE_SIZE, filePath, this.writeExecutor);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        this.segmentFile.shutdown();
+        this.writeExecutor.shutdown();
+        super.teardown();
+    }
+
+    @Test
+    public void testSwapInOut() throws Exception {
+        testWriteRead();
+        this.segmentFile.setReadOnly(true);
+        assertFalse(this.segmentFile.isSwappedOut());
+        this.segmentFile.swapOut();
+        assertTrue(this.segmentFile.isSwappedOut());
+
+        int firstWritePos = SegmentFile.HEADER_SIZE;
+
+        assertEquals(32, this.segmentFile.read(0, firstWritePos).length);
+        assertEquals(20, this.segmentFile.read(1, 38 + firstWritePos).length);
+        assertFalse(this.segmentFile.isSwappedOut());
+    }
+
+    @Test
+    public void testInitAndLoad() {
+        assertTrue(init());
+    }
+
+    private boolean init() {
+        SegmentFileOptions opts = SegmentFileOptions.builder() //
+            .setRecover(false) //
+            .setLastFile(true) //
+            .setNewFile(true) //
+            .setSync(true) //
+            .setPos(0).build();
+        return this.segmentFile.init(opts);
+    }
+
+    private byte[] genData(final int size) {
+        final byte[] bs = new byte[size];
+        ThreadLocalRandom.current().nextBytes(bs);
+        return bs;
+    }
+
+    @Test
+    public void testWriteRead() throws Exception {
+        init();
+        assertFalse(this.segmentFile.isFull());
+        int firstWritePos = SegmentFile.HEADER_SIZE;
+        assertNull(this.segmentFile.read(0, firstWritePos));
+        final byte[] data = genData(32);
+        assertFalse(this.segmentFile.reachesFileEndBy(SegmentFile.getWriteBytes(data)));
+        WriteContext events = new RocksDBSegmentLogStorage.BarrierWriteContext();
+        events.startJob();
+        assertEquals(firstWritePos, this.segmentFile.write(0, data, events));
+        events.joinAll();
+        // Can't read before sync
+        assertNull(this.segmentFile.read(0, firstWritePos));
+        this.segmentFile.sync(true);
+        assertArrayEquals(data, this.segmentFile.read(0, firstWritePos));
+        assertTrue(this.segmentFile.reachesFileEndBy(SegmentFile.getWriteBytes(data)));
+
+        final int nextWrotePos = 38 + SegmentFile.HEADER_SIZE;
+        assertEquals(nextWrotePos, this.segmentFile.getWrotePos());
+        assertEquals(nextWrotePos, this.segmentFile.getCommittedPos());
+        assertFalse(this.segmentFile.isFull());
+        final byte[] data2 = genData(20);
+        assertFalse(this.segmentFile.reachesFileEndBy(SegmentFile.getWriteBytes(data2)));
+        events = new RocksDBSegmentLogStorage.BarrierWriteContext();
+        events.startJob();
+        assertEquals(nextWrotePos, this.segmentFile.write(1, data2, events));
+        events.joinAll();
+        // Can't read before sync
+        assertNull(this.segmentFile.read(1, nextWrotePos));
+        this.segmentFile.sync(true);
+        assertArrayEquals(data2, this.segmentFile.read(1, nextWrotePos));
+        assertEquals(64 + SegmentFile.HEADER_SIZE, this.segmentFile.getWrotePos());
+        assertEquals(64 + SegmentFile.HEADER_SIZE, this.segmentFile.getCommittedPos());
+        assertTrue(this.segmentFile.isFull());
+    }
+
+    @Test
+    public void testRecoverFromDirtyMagic() throws Exception {
+        testWriteRead();
+        int firstWritePos = SegmentFile.HEADER_SIZE;
+
+        SegmentFileOptions opts = SegmentFileOptions.builder() //
+            .setRecover(true) //
+            .setLastFile(true) //
+            .setNewFile(false) //
+            .setSync(true) //
+            .setPos(0).build();
+        {
+            // Restart segment file, all data is valid.
+            this.segmentFile.shutdown();
+            assertTrue(this.segmentFile.init(opts));
+            assertEquals(32, this.segmentFile.read(0, firstWritePos).length);
+            assertEquals(20, this.segmentFile.read(1, 38 + firstWritePos).length);
+        }
+
+        {
+            // Corrupted magic bytes at pos=57
+            this.segmentFile.clear(39 + firstWritePos, true);
+            this.segmentFile.shutdown();
+            assertTrue(this.segmentFile.init(opts));
+            assertEquals(32, this.segmentFile.read(0, firstWritePos).length);
+            assertNull(this.segmentFile.read(1, 38 + firstWritePos));
+        }
+
+    }
+
+    @Test
+    public void testRecoverFromInvalidData() throws Exception {
+        testWriteRead();
+
+        SegmentFileOptions opts = SegmentFileOptions.builder() //
+            .setRecover(true) //
+            .setLastFile(true) //
+            .setNewFile(false) //
+            .setSync(true) //
+            .setPos(0).build();
+        int firstWritePos = SegmentFile.HEADER_SIZE;
+        {
+            // Restart segment file, all data is valid.
+            this.segmentFile.shutdown();
+            assertTrue(this.segmentFile.init(opts));
+            assertEquals(32, this.segmentFile.read(0, firstWritePos).length);
+            assertEquals(20, this.segmentFile.read(1, firstWritePos + 38).length);
+        }
+
+        {
+            this.segmentFile.shutdown();
+            try (FileOutputStream out = new FileOutputStream(new File(this.segmentFile.getPath()), true);
+                    FileChannel outChan = out.getChannel()) {
+                // Cleared data after pos=62, the second data will be truncated when recovering.
+                outChan.truncate(44 + SegmentFile.HEADER_SIZE);
+            }
+            assertTrue(this.segmentFile.init(opts));
+            // First data is still valid
+            assertEquals(32, this.segmentFile.read(0, firstWritePos).length);
+            // The second data is truncated.
+            assertNull(this.segmentFile.read(1, 38 + firstWritePos));
+        }
+
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottleTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottleTest.java
new file mode 100644
index 0000000..9b2f6c4
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/ThroughputSnapshotThrottleTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ThroughputSnapshotThrottleTest {
+    private ThroughputSnapshotThrottle snapshotThrottle;
+
+    @Before
+    public void setup() {
+        //1K every seconds
+        this.snapshotThrottle = new ThroughputSnapshotThrottle(1024, 1);
+    }
+
+    @Test
+    public void testThrottledByThroughput() throws Exception {
+        assertEquals(512L, this.snapshotThrottle.throttledByThroughput(512));
+        assertEquals(512L, this.snapshotThrottle.throttledByThroughput(512));
+        assertEquals(0, this.snapshotThrottle.throttledByThroughput(512));
+        Thread.sleep(1100);
+        assertEquals(512L, this.snapshotThrottle.throttledByThroughput(512));
+        assertEquals(512L, this.snapshotThrottle.throttledByThroughput(512));
+        assertEquals(0, this.snapshotThrottle.throttledByThroughput(512));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopierTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopierTest.java
new file mode 100644
index 0000000..fba2d16
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotCopierTest.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.nio.ByteBuffer;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.Scheduler;
+import com.alipay.sofa.jraft.core.TimerManager;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.CopyOptions;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcResponseClosure;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class LocalSnapshotCopierTest extends BaseStorageTest {
+    private LocalSnapshotCopier    copier;
+    @Mock
+    private RaftClientService      raftClientService;
+    private String                 uri;
+    private final String           hostPort = "localhost:8081";
+    private final int              readerId = 99;
+    private CopyOptions            copyOpts;
+    private LocalSnapshotMetaTable table;
+    private LocalSnapshotWriter    writer;
+    private LocalSnapshotReader    reader;
+    private RaftOptions            raftOptions;
+    @Mock
+    private LocalSnapshotStorage   snapshotStorage;
+    private Scheduler              timerManager;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.timerManager = new TimerManager(5);
+        this.raftOptions = new RaftOptions();
+        this.writer = new LocalSnapshotWriter(this.path, this.snapshotStorage, this.raftOptions);
+        this.reader = new LocalSnapshotReader(this.snapshotStorage, null, new Endpoint("localhost", 8081),
+            this.raftOptions, this.path);
+
+        Mockito.when(this.snapshotStorage.open()).thenReturn(this.reader);
+        Mockito.when(this.snapshotStorage.create(true)).thenReturn(this.writer);
+
+        this.table = new LocalSnapshotMetaTable(this.raftOptions);
+        this.table.addFile("testFile", LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test").build());
+        this.table.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(1).build());
+        this.uri = "remote://" + this.hostPort + "/" + this.readerId;
+        this.copier = new LocalSnapshotCopier();
+        this.copyOpts = new CopyOptions();
+        Mockito.when(this.raftClientService.connect(new Endpoint("localhost", 8081))).thenReturn(true);
+        assertTrue(this.copier.init(this.uri, new SnapshotCopierOptions(this.raftClientService, this.timerManager,
+            this.raftOptions, new NodeOptions())));
+        this.copier.setStorage(this.snapshotStorage);
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        super.teardown();
+        this.copier.close();
+        this.timerManager.shutdown();
+    }
+
+    @Test
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public void testCancelByRemote() throws Exception {
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        final ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+        this.copier.start();
+        Thread.sleep(500);
+        final RpcResponseClosure closure = argument.getValue();
+
+        closure.run(new Status(RaftError.ECANCELED, "test cancel"));
+
+        this.copier.join();
+        //start timer
+        final SnapshotReader reader = this.copier.getReader();
+        assertNull(reader);
+        Assert.assertEquals(RaftError.ECANCELED.getNumber(), this.copier.getCode());
+        Assert.assertEquals("test cancel", this.copier.getErrorMsg());
+    }
+
+    @Test
+    public void testInterrupt() throws Exception {
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        final ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+        this.copier.start();
+        Thread.sleep(10);
+
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                LocalSnapshotCopierTest.this.copier.cancel();
+            }
+        });
+        this.copier.join();
+        //start timer
+        final SnapshotReader reader = this.copier.getReader();
+        assertNull(reader);
+        Assert.assertEquals(RaftError.ECANCELED.getNumber(), this.copier.getCode());
+        Assert.assertEquals("Cancel the copier manually.", this.copier.getErrorMsg());
+    }
+
+    @Test
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    public void testStartJoinFinishOK() throws Exception {
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename(Snapshot.JRAFT_SNAPSHOT_META_FILE).setCount(Integer.MAX_VALUE).setOffset(0)
+            .setReadPartly(true);
+
+        //mock get metadata
+        ArgumentCaptor argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+        this.copier.start();
+        Thread.sleep(500);
+        RpcResponseClosure closure = argument.getValue();
+        final ByteBuffer metaBuf = this.table.saveToByteBufferAsRemote();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(metaBuf.remaining()).setEof(true)
+            .setData(ByteString.copyFrom(metaBuf)).build());
+
+        //mock get file
+        argument = ArgumentCaptor.forClass(RpcResponseClosure.class);
+        rb.setFilename("testFile");
+        rb.setCount(this.raftOptions.getMaxByteCountPerRpc());
+        Mockito.when(
+            this.raftClientService.getFile(eq(new Endpoint("localhost", 8081)), eq(rb.build()),
+                eq(this.copyOpts.getTimeoutMs()), argument.capture())).thenReturn(future);
+
+        closure.run(Status.OK());
+
+        Thread.sleep(500);
+        closure = argument.getValue();
+        closure.setResponse(RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
+            .setData(ByteString.copyFrom(new byte[100])).build());
+        closure.run(Status.OK());
+        this.copier.join();
+        final SnapshotReader reader = this.copier.getReader();
+        assertSame(this.reader, reader);
+        assertEquals(1, this.writer.listFiles().size());
+        assertTrue(this.writer.listFiles().contains("testFile"));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTableTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTableTest.java
new file mode 100644
index 0000000..8438c3a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotMetaTableTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class LocalSnapshotMetaTableTest {
+    private LocalSnapshotMetaTable table;
+
+    @Before
+    public void setup() {
+        this.table = new LocalSnapshotMetaTable(new RaftOptions());
+    }
+
+    @Test
+    public void testAddRemove() {
+        LocalFileMetaOutter.LocalFileMeta meta = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertEquals(0, table.listFiles().size());
+        assertTrue(this.table.addFile("data", meta));
+        assertFalse(this.table.addFile("data", meta));
+
+        assertEquals(1, table.listFiles().size());
+        assertTrue(table.listFiles().contains("data"));
+
+        assertTrue(this.table.removeFile("data"));
+        assertFalse(this.table.removeFile("data"));
+        assertEquals(0, table.listFiles().size());
+    }
+
+    @Test
+    public void testSaveLoadFile() throws IOException {
+        LocalFileMetaOutter.LocalFileMeta meta1 = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("data1")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertTrue(this.table.addFile("data1", meta1));
+        LocalFileMetaOutter.LocalFileMeta meta2 = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("data2")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertTrue(this.table.addFile("data2", meta2));
+
+        assertTrue(table.listFiles().contains("data1"));
+        assertTrue(table.listFiles().contains("data2"));
+
+        String path = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(path));
+        try {
+            String filePath = path + File.separator + "table";
+            table.saveToFile(filePath);
+
+            LocalSnapshotMetaTable newTable = new LocalSnapshotMetaTable(new RaftOptions());
+            assertNull(newTable.getFileMeta("data1"));
+            assertNull(newTable.getFileMeta("data2"));
+            assertTrue(newTable.loadFromFile(filePath));
+            Assert.assertEquals(meta1, newTable.getFileMeta("data1"));
+            Assert.assertEquals(meta2, newTable.getFileMeta("data2"));
+        } finally {
+            FileUtils.deleteDirectory(new File(path));
+        }
+    }
+
+    @Test
+    public void testSaveLoadIoBuffer() throws Exception {
+        LocalFileMetaOutter.LocalFileMeta meta1 = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("data1")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertTrue(this.table.addFile("data1", meta1));
+        LocalFileMetaOutter.LocalFileMeta meta2 = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("data2")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertTrue(this.table.addFile("data2", meta2));
+
+        ByteBuffer buf = this.table.saveToByteBufferAsRemote();
+        assertNotNull(buf);
+        assertTrue(buf.hasRemaining());
+
+        LocalSnapshotMetaTable newTable = new LocalSnapshotMetaTable(new RaftOptions());
+        assertNull(newTable.getFileMeta("data1"));
+        assertNull(newTable.getFileMeta("data2"));
+        assertTrue(newTable.loadFromIoBufferAsRemote(buf));
+        Assert.assertEquals(meta1, newTable.getFileMeta("data1"));
+        Assert.assertEquals(meta2, newTable.getFileMeta("data2"));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReaderTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReaderTest.java
new file mode 100644
index 0000000..834803c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotReaderTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.FileService;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class LocalSnapshotReaderTest extends BaseStorageTest {
+
+    private LocalSnapshotReader    reader;
+    @Mock
+    private LocalSnapshotStorage   snapshotStorage;
+    private LocalSnapshotMetaTable table;
+    private final int              snapshotIndex = 99;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.path = this.path + File.separator + Snapshot.JRAFT_SNAPSHOT_PREFIX + snapshotIndex;
+        FileUtils.forceMkdir(new File(path));
+        this.table = new LocalSnapshotMetaTable(new RaftOptions());
+        this.table.addFile("testFile", LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test").build());
+        table.saveToFile(path + File.separator + Snapshot.JRAFT_SNAPSHOT_META_FILE);
+        this.reader = new LocalSnapshotReader(snapshotStorage, null, new Endpoint("localhost", 8081),
+            new RaftOptions(), path);
+        assertTrue(this.reader.init(null));
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        super.teardown();
+        this.reader.close();
+        Mockito.verify(this.snapshotStorage, Mockito.only()).unref(this.snapshotIndex);
+        assertFalse(FileService.getInstance().removeReader(this.reader.getReaderId()));
+    }
+
+    @Test
+    public void testListFiles() {
+        assertTrue(this.reader.listFiles().contains("testFile"));
+    }
+
+    @Test
+    public void testGenerateUriForCopy() throws Exception {
+        final String uri = this.reader.generateURIForCopy();
+        assertNotNull(uri);
+        assertTrue(uri.startsWith("remote://localhost:8081/"));
+        final long readerId = Long.valueOf(uri.substring("remote://localhost:8081/".length()));
+        assertTrue(FileService.getInstance().removeReader(readerId));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorageTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorageTest.java
new file mode 100644
index 0000000..5d4077f
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotStorageTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.File;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class LocalSnapshotStorageTest extends BaseStorageTest {
+    private LocalSnapshotStorage   snapshotStorage;
+    private LocalSnapshotMetaTable table;
+    private int                    lastSnapshotIndex = 99;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+
+        String snapshotPath = this.path + File.separator + Snapshot.JRAFT_SNAPSHOT_PREFIX + lastSnapshotIndex;
+        FileUtils.forceMkdir(new File(snapshotPath));
+        this.table = new LocalSnapshotMetaTable(new RaftOptions());
+        this.table.setMeta(RaftOutter.SnapshotMeta.newBuilder().setLastIncludedIndex(this.lastSnapshotIndex)
+            .setLastIncludedTerm(1).build());
+        this.table.saveToFile(snapshotPath + File.separator + Snapshot.JRAFT_SNAPSHOT_META_FILE);
+
+        this.snapshotStorage = new LocalSnapshotStorage(path, new RaftOptions());
+        assertTrue(this.snapshotStorage.init(null));
+    }
+
+    @Override
+    @After
+    public void teardown() throws Exception {
+        super.teardown();
+        this.snapshotStorage.shutdown();
+    }
+
+    @Test
+    public void testGetLastSnapshotIndex() throws Exception {
+        assertEquals(this.snapshotStorage.getLastSnapshotIndex(), lastSnapshotIndex);
+        assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex).get());
+    }
+
+    @Test
+    public void testCreateOpen() throws Exception {
+        SnapshotWriter writer = this.snapshotStorage.create();
+        assertNotNull(writer);
+        RaftOutter.SnapshotMeta wroteMeta = RaftOutter.SnapshotMeta.newBuilder()
+            .setLastIncludedIndex(this.lastSnapshotIndex + 1).setLastIncludedTerm(1).build();
+        ((LocalSnapshotWriter) writer).saveMeta(wroteMeta);
+        writer.addFile("data");
+        assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex).get());
+        writer.close();
+        //release old
+        assertEquals(0, this.snapshotStorage.getRefs(this.lastSnapshotIndex).get());
+        //ref new
+        assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
+        SnapshotReader reader = this.snapshotStorage.open();
+        assertNotNull(reader);
+        assertTrue(reader.listFiles().contains("data"));
+        RaftOutter.SnapshotMeta readMeta = reader.load();
+        assertEquals(wroteMeta, readMeta);
+        assertEquals(2, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
+        reader.close();
+        assertEquals(1, this.snapshotStorage.getRefs(this.lastSnapshotIndex + 1).get());
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriterTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriterTest.java
new file mode 100644
index 0000000..ddfefd4
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/LocalSnapshotWriterTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class LocalSnapshotWriterTest extends BaseStorageTest {
+    private LocalSnapshotWriter  writer;
+    @Mock
+    private LocalSnapshotStorage snapshotStorage;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.writer = new LocalSnapshotWriter(path, snapshotStorage, new RaftOptions());
+        assertTrue(this.writer.init(null));
+    }
+
+    @Test
+    public void testAddRemove() throws Exception {
+        assertNull(this.writer.getFileMeta("data"));
+        assertFalse(this.writer.listFiles().contains("data"));
+        assertTrue(this.writer.addFile("data"));
+        assertFalse(this.writer.addFile("data"));
+        assertTrue(this.writer.listFiles().contains("data"));
+        assertNotNull(this.writer.getFileMeta("data"));
+        assertTrue(this.writer.removeFile("data"));
+        assertFalse(this.writer.removeFile("data"));
+        assertNull(this.writer.getFileMeta("data"));
+        assertFalse(this.writer.listFiles().contains("data"));
+    }
+
+    @Test
+    public void testSyncInit() throws Exception {
+        LocalFileMetaOutter.LocalFileMeta meta = LocalFileMetaOutter.LocalFileMeta.newBuilder().setChecksum("test")
+            .setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        assertTrue(this.writer.addFile("data1", meta));
+        assertTrue(this.writer.addFile("data2"));
+
+        assertEquals(meta, this.writer.getFileMeta("data1"));
+        assertFalse(((LocalFileMetaOutter.LocalFileMeta) this.writer.getFileMeta("data2")).hasChecksum());
+        assertFalse(((LocalFileMetaOutter.LocalFileMeta) this.writer.getFileMeta("data2")).hasUserMeta());
+
+        this.writer.sync();
+        //create a new writer
+        LocalSnapshotWriter newWriter = new LocalSnapshotWriter(path, snapshotStorage, new RaftOptions());
+        assertTrue(newWriter.init(null));
+        assertNotSame(writer, newWriter);
+        assertEquals(meta, newWriter.getFileMeta("data1"));
+        assertFalse(((LocalFileMetaOutter.LocalFileMeta) newWriter.getFileMeta("data2")).hasChecksum());
+        assertFalse(((LocalFileMetaOutter.LocalFileMeta) newWriter.getFileMeta("data2")).hasUserMeta());
+    }
+
+    @Test
+    public void testClose() throws Exception {
+        this.writer.close();
+        Mockito.verify(this.snapshotStorage, Mockito.only()).close(this.writer, false);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReaderTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReaderTest.java
new file mode 100644
index 0000000..08179fb
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/local/SnapshotFileReaderTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.local;
+
+import java.io.FileNotFoundException;
+import java.nio.ByteBuffer;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.storage.BaseStorageTest;
+import com.alipay.sofa.jraft.storage.snapshot.Snapshot;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class SnapshotFileReaderTest extends BaseStorageTest {
+    private SnapshotFileReader     reader;
+    private LocalSnapshotMetaTable metaTable;
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+        this.reader = new SnapshotFileReader(path, null);
+        metaTable = new LocalSnapshotMetaTable(new RaftOptions());
+        this.reader.setMetaTable(metaTable);
+    }
+
+    @Test
+    public void testReadMetaFile() throws Exception {
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(1024);
+        final LocalFileMetaOutter.LocalFileMeta meta = addDataMeta();
+        assertEquals(-1, this.reader.readFile(bufRef, Snapshot.JRAFT_SNAPSHOT_META_FILE, 0, Integer.MAX_VALUE));
+
+        final ByteBuffer buf = bufRef.getBuffer();
+        buf.flip();
+        final LocalSnapshotMetaTable newTable = new LocalSnapshotMetaTable(new RaftOptions());
+        newTable.loadFromIoBufferAsRemote(buf);
+        Assert.assertEquals(meta, newTable.getFileMeta("data"));
+    }
+
+    private LocalFileMetaOutter.LocalFileMeta addDataMeta() {
+        final LocalFileMetaOutter.LocalFileMeta meta = LocalFileMetaOutter.LocalFileMeta.newBuilder()
+            .setChecksum("test").setSource(LocalFileMetaOutter.FileSource.FILE_SOURCE_LOCAL).build();
+        this.metaTable.addFile("data", meta);
+        return meta;
+    }
+
+    @Test
+    public void testReadFile() throws Exception {
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate();
+        try {
+            this.reader.readFile(bufRef, "unfound", 0, 1024);
+            fail();
+        } catch (final FileNotFoundException e) {
+
+        }
+
+        final String data = writeData();
+        addDataMeta();
+
+        final int read = this.reader.readFile(bufRef, "data", 0, 1024);
+        assertEquals(-1, read);
+        final ByteBuffer buf = bufRef.getBuffer();
+        buf.flip();
+        assertEquals(data.length(), buf.remaining());
+        final byte[] bs = new byte[data.length()];
+        buf.get(bs);
+        assertEquals(data, new String(bs));
+
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySessionTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySessionTest.java
new file mode 100644
index 0000000..8d375e3
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/CopySessionTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.remote;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.TimerManager;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.CopyOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.impl.FutureImpl;
+import com.alipay.sofa.jraft.util.ByteBufferCollector;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Utils;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class CopySessionTest {
+    private CopySession                        session;
+    @Mock
+    private RaftClientService                  rpcService;
+    private RpcRequests.GetFileRequest.Builder rb;
+    private final Endpoint                     address = new Endpoint("localhost", 8081);
+    private CopyOptions                        copyOpts;
+    private RaftOptions                        raftOpts;
+    private TimerManager                       timerManager;
+
+    @Before
+    public void setup() {
+        this.timerManager = new TimerManager(5);
+        this.copyOpts = new CopyOptions();
+        this.rb = RpcRequests.GetFileRequest.newBuilder();
+        this.rb.setReaderId(99);
+        this.rb.setFilename("data");
+        this.raftOpts = new RaftOptions();
+        this.session = new CopySession(rpcService, timerManager, null, raftOpts, rb, address);
+        this.session.setCopyOptions(copyOpts);
+    }
+
+    @After
+    public void teardown() {
+        Utils.closeQuietly(this.session);
+        this.timerManager.shutdown();
+    }
+
+    @Test
+    public void testSendNextRpc() {
+        final int maxCount = this.raftOpts.getMaxByteCountPerRpc();
+        sendNextRpc(maxCount);
+    }
+
+    @Test
+    public void testSendNextRpcWithBuffer() {
+        session.setDestBuf(ByteBufferCollector.allocate(1));
+        final int maxCount = Integer.MAX_VALUE;
+        sendNextRpc(maxCount);
+    }
+
+    @Test
+    public void testOnRpcReturnedEOF() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread() {
+            @Override
+            public void run() {
+                try {
+                    //test join, should return
+                    session.join();
+                    latch.countDown();
+                } catch (final InterruptedException e) {
+
+                }
+            }
+        }.start();
+        assertNull(this.session.getRpcCall());
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(0);
+        this.session.setDestBuf(bufRef);
+
+        this.session.onRpcReturned(Status.OK(), RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(true)
+            .setData(ByteString.copyFrom(new byte[100])).build());
+        assertEquals(100, bufRef.capacity());
+        //should be flip
+        assertEquals(0, bufRef.getBuffer().position());
+        assertEquals(100, bufRef.getBuffer().remaining());
+
+        assertNull(this.session.getRpcCall());
+        latch.await();
+    }
+
+    @Test
+    public void testOnRpcReturnedOK() {
+        assertNull(this.session.getRpcCall());
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(0);
+        this.session.setDestBuf(bufRef);
+
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename("data").setCount(Integer.MAX_VALUE).setOffset(100).setReadPartly(true);
+        Mockito
+            .when(this.rpcService.getFile(this.address, rb.build(), this.copyOpts.getTimeoutMs(), session.getDone()))
+            .thenReturn(future);
+
+        this.session.onRpcReturned(Status.OK(), RpcRequests.GetFileResponse.newBuilder().setReadSize(100).setEof(false)
+            .setData(ByteString.copyFrom(new byte[100])).build());
+        assertEquals(100, bufRef.capacity());
+        assertEquals(100, bufRef.getBuffer().position());
+
+        assertNotNull(this.session.getRpcCall());
+        //send next request
+        assertSame(future, this.session.getRpcCall());
+    }
+
+    @Test
+    public void testOnRpcReturnedRetry() throws Exception {
+        assertNull(this.session.getTimer());
+        assertNull(this.session.getRpcCall());
+        final ByteBufferCollector bufRef = ByteBufferCollector.allocate(0);
+        this.session.setDestBuf(bufRef);
+
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename("data").setCount(Integer.MAX_VALUE).setOffset(0).setReadPartly(true);
+        Mockito
+            .when(this.rpcService.getFile(this.address, rb.build(), this.copyOpts.getTimeoutMs(), session.getDone()))
+            .thenReturn(future);
+
+        this.session.onRpcReturned(new Status(RaftError.EINTR, "test"), null);
+        assertNotNull(this.session.getTimer());
+        Thread.sleep(this.copyOpts.getRetryIntervalMs() + 100);
+        assertNotNull(this.session.getRpcCall());
+        assertSame(future, this.session.getRpcCall());
+        assertNull(this.session.getTimer());
+    }
+
+    private void sendNextRpc(int maxCount) {
+        assertNull(this.session.getRpcCall());
+        final FutureImpl future = new FutureImpl<>();
+        final RpcRequests.GetFileRequest.Builder rb = RpcRequests.GetFileRequest.newBuilder().setReaderId(99)
+            .setFilename("data").setCount(maxCount).setOffset(0).setReadPartly(true);
+        Mockito
+            .when(this.rpcService.getFile(this.address, rb.build(), this.copyOpts.getTimeoutMs(), session.getDone()))
+            .thenReturn(future);
+        this.session.sendNextRpc();
+        assertNotNull(this.session.getRpcCall());
+        assertSame(future, this.session.getRpcCall());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopierTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopierTest.java
new file mode 100644
index 0000000..5b625d9
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/storage/snapshot/remote/RemoteFileCopierTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.storage.snapshot.remote;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.core.TimerManager;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.option.SnapshotCopierOptions;
+import com.alipay.sofa.jraft.rpc.RaftClientService;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class RemoteFileCopierTest {
+    private RemoteFileCopier  copier;
+    @Mock
+    private RaftClientService rpcService;
+    private TimerManager      timerManager;
+
+    @Before
+    public void setup() {
+        this.timerManager = new TimerManager(5);
+        copier = new RemoteFileCopier();
+    }
+
+    @Test
+    public void testInit() {
+        Mockito.when(rpcService.connect(new Endpoint("localhost", 8081))).thenReturn(true);
+        assertTrue(copier.init("remote://localhost:8081/999", null, new SnapshotCopierOptions(rpcService, timerManager,
+            new RaftOptions(), new NodeOptions())));
+        assertEquals(999, copier.getReaderId());
+        Assert.assertEquals("localhost", copier.getEndpoint().getIp());
+        Assert.assertEquals(8081, copier.getEndpoint().getPort());
+    }
+
+    @Test
+    public void testInitFail() {
+        Mockito.when(rpcService.connect(new Endpoint("localhost", 8081))).thenReturn(false);
+        assertFalse(copier.init("remote://localhost:8081/999", null, new SnapshotCopierOptions(rpcService,
+            timerManager, new RaftOptions(), new NodeOptions())));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java
new file mode 100644
index 0000000..0f07dae
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test;
+
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.google.protobuf.Message;
+
+/**
+ * mock alipay remoting async context
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-19 6:05:46 PM
+ */
+public class MockAsyncContext implements RpcContext {
+    private Object responseObject;
+
+    public Object getResponseObject() {
+        return this.responseObject;
+    }
+
+    @SuppressWarnings("unchecked")
+    public  T as(Class t) {
+        return (T) this.responseObject;
+    }
+
+    public void setResponseObject(Object responseObject) {
+        this.responseObject = responseObject;
+    }
+
+    @Override
+    public void sendResponse(Object responseObject) {
+        this.responseObject = responseObject;
+
+    }
+
+    @Override
+    public Connection getConnection() {
+        return null;
+    }
+
+    @Override
+    public String getRemoteAddress() {
+        return "localhost:12345";
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java
new file mode 100644
index 0000000..939b0de
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Test helper
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ *         2018-Apr-11 10:16:07 AM
+ */
+public class TestUtils {
+
+    public static ConfigurationEntry getConfEntry(final String confStr, final String oldConfStr) {
+        ConfigurationEntry entry = new ConfigurationEntry();
+        entry.setConf(JRaftUtils.getConfiguration(confStr));
+        entry.setOldConf(JRaftUtils.getConfiguration(oldConfStr));
+        return entry;
+    }
+
+    public static void dumpThreads() {
+        try {
+            ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+            ThreadInfo[] infos = bean.dumpAllThreads(true, true);
+            for (ThreadInfo info : infos) {
+                System.out.println(info);
+            }
+        } catch (Throwable t) {
+            t.printStackTrace(); // NOPMD
+        }
+    }
+
+    public static String mkTempDir() {
+        return Paths.get(System.getProperty("java.io.tmpdir", "/tmp"), "jraft_test_" + System.nanoTime()).toString();
+    }
+
+    public static LogEntry mockEntry(final int index, final int term) {
+        return mockEntry(index, term, 0);
+    }
+
+    public static LogEntry mockEntry(final int index, final int term, final int dataSize) {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(index, term));
+        if (dataSize > 0) {
+            byte[] bs = new byte[dataSize];
+            ThreadLocalRandom.current().nextBytes(bs);
+            entry.setData(ByteBuffer.wrap(bs));
+        }
+        return entry;
+    }
+
+    public static List mockEntries() {
+        return mockEntries(10);
+    }
+
+    public static String getMyIp() {
+        String ip = null;
+        try {
+            Enumeration interfaces = NetworkInterface.getNetworkInterfaces();
+            while (interfaces.hasMoreElements()) {
+                NetworkInterface iface = interfaces.nextElement();
+                // filters out 127.0.0.1 and inactive interfaces
+                if (iface.isLoopback() || !iface.isUp()) {
+                    continue;
+                }
+
+                Enumeration addresses = iface.getInetAddresses();
+                while (addresses.hasMoreElements()) {
+                    InetAddress addr = addresses.nextElement();
+                    if (addr instanceof Inet4Address) {
+                        ip = addr.getHostAddress();
+                        break;
+                    }
+                }
+            }
+            return ip;
+        } catch (SocketException e) {
+            return "localhost";
+        }
+    }
+
+    public static List mockEntries(final int n) {
+        List entries = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            LogEntry entry = mockEntry(i, i);
+            if (i > 0) {
+                entry.setData(ByteBuffer.wrap(String.valueOf(i).getBytes()));
+            }
+            entries.add(entry);
+        }
+        return entries;
+    }
+
+    public static RpcRequests.PingRequest createPingRequest() {
+        RpcRequests.PingRequest reqObject = RpcRequests.PingRequest.newBuilder()
+            .setSendTimestamp(System.currentTimeMillis()).build();
+        return reqObject;
+    }
+
+    public static final int INIT_PORT = 5003;
+
+    public static List generatePeers(final int n) {
+        List ret = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            ret.add(new PeerId(getMyIp(), INIT_PORT + i));
+        }
+        return ret;
+    }
+
+    public static List generatePriorityPeers(final int n, final List priorities) {
+        List ret = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            Endpoint endpoint = new Endpoint(getMyIp(), INIT_PORT + i);
+            PeerId peerId = new PeerId(endpoint, 0, priorities.get(i));
+            ret.add(peerId);
+        }
+        return ret;
+    }
+
+    public static byte[] getRandomBytes() {
+        final byte[] requestContext = new byte[ThreadLocalRandom.current().nextInt(10) + 1];
+        ThreadLocalRandom.current().nextBytes(requestContext);
+        return requestContext;
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocatorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocatorTest.java
new file mode 100644
index 0000000..aee9796
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AdaptiveBufAllocatorTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class AdaptiveBufAllocatorTest {
+
+    private AdaptiveBufAllocator.Handle handle;
+
+    /*
+     * The allocate size table:
+     *
+     * [16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 272, 288, 304, 320, 336, 352, 368,
+     * 384, 400, 416, 432, 448, 464, 480, 496, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072, 262144,
+     * 524288, 1048576, 2097152, 4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912,
+     * 1073741824]
+     */
+
+    @Before
+    public void setup() {
+        this.handle = new AdaptiveBufAllocator(64, 512, 524288).newHandle();
+    }
+
+    @Test
+    public void incrementTest() {
+        allocReadExpected(this.handle, 512);
+        allocReadExpected(this.handle, 8192);
+        allocReadExpected(this.handle, 131072);
+        allocReadExpected(this.handle, 524288);
+        allocRead(this.handle, 524288, 8388608);
+    }
+
+    @Test
+    public void decreaseTest() {
+        allocRead(this.handle, 512, 16);
+        allocRead(this.handle, 512, 16);
+        allocRead(this.handle, 496, 16);
+        allocRead(this.handle, 496, 16);
+        allocRead(this.handle, 480, 16);
+        allocRead(this.handle, 480, 16);
+        allocRead(this.handle, 464, 16);
+        allocRead(this.handle, 464, 16);
+    }
+
+    private static void allocReadExpected(final AdaptiveBufAllocator.Handle handle, final int expectedSize) {
+        allocRead(handle, expectedSize, expectedSize);
+    }
+
+    private static void allocRead(final AdaptiveBufAllocator.Handle handle, final int expectedSize, final int lastRead) {
+        assertEquals(expectedSize, handle.allocate().capacity());
+        handle.record(lastRead);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ArrayDequeTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ArrayDequeTest.java
new file mode 100644
index 0000000..4cfae04
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ArrayDequeTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class ArrayDequeTest {
+
+    @Test
+    public void testPeekPoll() {
+        ArrayDeque list = new ArrayDeque<>();
+        for (int i = 0; i < 10; i++) {
+            list.add(i);
+        }
+        assertEquals(0, (int) list.peekFirst());
+        assertEquals(9, (int) list.peekLast());
+        assertEquals(0, (int) list.peekFirst());
+        assertEquals(9, (int) list.peekLast());
+        assertEquals(10, list.size());
+
+        for (int i = 0; i < 10; i++) {
+            assertEquals(i, (int) list.get(i));
+        }
+
+        assertEquals(0, (int) list.pollFirst());
+        assertEquals(1, (int) list.peekFirst());
+        assertEquals(9, (int) list.pollLast());
+        assertEquals(8, (int) list.peekLast());
+
+        assertEquals(1, (int) list.pollFirst());
+        assertEquals(2, (int) list.peekFirst());
+        assertEquals(8, (int) list.pollLast());
+        assertEquals(7, (int) list.peekLast());
+
+        while (!list.isEmpty()) {
+            list.pollFirst();
+        }
+
+        try {
+            list.pollFirst();
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+            assertTrue(true);
+        }
+
+        try {
+            list.pollLast();
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+            assertTrue(true);
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiCodecBenchmark.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiCodecBenchmark.java
new file mode 100644
index 0000000..be2162e
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiCodecBenchmark.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+/**
+ * @author jiachun.fjc
+ */
+@SuppressWarnings({ "all" })
+@State(Scope.Benchmark)
+public class AsciiCodecBenchmark {
+
+    /*
+        Benchmark                              Mode  Cnt  Score   Error   Units
+        AsciiCodecBenchmark.fastpathDecode    thrpt    3  0.087 ± 0.024  ops/ns
+        AsciiCodecBenchmark.fastpathEncode    thrpt    3  0.093 ± 0.047  ops/ns
+        AsciiCodecBenchmark.normalpathDecode  thrpt    3  0.020 ± 0.006  ops/ns
+        AsciiCodecBenchmark.normalpathEncode  thrpt    3  0.017 ± 0.032  ops/ns
+     */
+
+    private static final String PEER_STR   = "127.0.0.1:18090:1";
+    private static final byte[] PEER_BYTES = PEER_STR.getBytes();
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void normalpathEncode() {
+        PEER_STR.getBytes();
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void normalpathDecode() {
+        new String(PEER_BYTES);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void fastpathEncode() {
+        // fast ptah
+        AsciiStringUtil.unsafeEncode(PEER_STR);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void fastpathDecode() {
+        // fast ptah
+        AsciiStringUtil.unsafeDecode(PEER_BYTES);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(AsciiCodecBenchmark.class.getSimpleName()) //
+            .warmupIterations(3) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiStringUtilTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiStringUtilTest.java
new file mode 100644
index 0000000..c63024c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/AsciiStringUtilTest.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author jiachun.fjc
+ */
+public class AsciiStringUtilTest {
+
+    @Test
+    public void codecTest() {
+        final String asciiText = "127.0.0.1:8080";
+        final byte[] bytes1 = AsciiStringUtil.unsafeEncode(asciiText);
+        final byte[] bytes2 = asciiText.getBytes();
+        Assert.assertArrayEquals(bytes1, bytes2);
+
+        final String s1 = new String(bytes1);
+        final String s2 = AsciiStringUtil.unsafeDecode(bytes2);
+        Assert.assertEquals(s1, s2);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BitsTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BitsTest.java
new file mode 100644
index 0000000..15a7e6d
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BitsTest.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class BitsTest {
+
+    @Test
+    public void testGetSet() {
+        byte[] bs = new byte[1 + 2 + 4 + 8];
+
+        bs[0] = (byte) 1;
+        Bits.putShort(bs, 1, (short) 2);
+        Bits.putInt(bs, 3, 3);
+        Bits.putLong(bs, 7, 99L);
+
+        assertEquals(1, bs[0]);
+        assertEquals((short) 2, Bits.getShort(bs, 1));
+        assertEquals(3, Bits.getInt(bs, 3));
+        assertEquals(99L, Bits.getLong(bs, 7));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ByteBufferCollectorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ByteBufferCollectorTest.java
new file mode 100644
index 0000000..e1709ed
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ByteBufferCollectorTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ByteBufferCollectorTest {
+
+    @Test
+    public void testSmallSizeRecycle() {
+        final ByteBufferCollector object = ByteBufferCollector.allocateByRecyclers(4096);
+        object.recycle();
+        assertEquals(4096, ByteBufferCollector.allocateByRecyclers().capacity());
+    }
+
+    @Test
+    public void testLargeSizeRecycle() {
+        final ByteBufferCollector object = ByteBufferCollector.allocateByRecyclers(4 * 1024 * 1024 + 1);
+        object.recycle();
+        assertEquals(Utils.RAFT_DATA_BUF_SIZE, ByteBufferCollector.allocateByRecyclers().capacity());
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testMultipleRecycle() {
+        final ByteBufferCollector object = ByteBufferCollector.allocateByRecyclers();
+        object.recycle();
+        object.recycle();
+    }
+
+    @Test
+    public void testMultipleRecycleAtDifferentThread() throws InterruptedException {
+        final ByteBufferCollector object = ByteBufferCollector.allocateByRecyclers();
+        final Thread thread1 = new Thread(object::recycle);
+        thread1.start();
+        thread1.join();
+        assertSame(object, ByteBufferCollector.allocateByRecyclers());
+    }
+
+    @Test
+    public void testRecycle() {
+        final ByteBufferCollector object = ByteBufferCollector.allocateByRecyclers();
+        object.recycle();
+        final ByteBufferCollector object2 = ByteBufferCollector.allocateByRecyclers();
+        Assert.assertSame(object, object2);
+        object2.recycle();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BytesUtilTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BytesUtilTest.java
new file mode 100644
index 0000000..5296498
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/BytesUtilTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+public class BytesUtilTest {
+
+    @Test
+    public void testNullToEmpty() {
+        Assert.assertArrayEquals(new byte[] {}, BytesUtil.nullToEmpty(null));
+        Assert.assertArrayEquals(new byte[] { 1, 2 }, BytesUtil.nullToEmpty(new byte[] { 1, 2 }));
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    @Test
+    public void testIsEmpty() {
+        Assert.assertTrue(BytesUtil.isEmpty(null));
+
+        Assert.assertFalse(BytesUtil.isEmpty(new byte[] { 1, 2 }));
+    }
+
+    @Test
+    public void testWriteUtf8() {
+        Assert.assertNull(BytesUtil.writeUtf8(null));
+
+        Assert.assertArrayEquals(new byte[] { 102, 111, 111 }, BytesUtil.writeUtf8("foo"));
+    }
+
+    @Test
+    public void testReadUtf8() {
+        Assert.assertNull(BytesUtil.readUtf8(null));
+
+        Assert.assertEquals("foo", BytesUtil.readUtf8(new byte[] { 102, 111, 111 }));
+    }
+
+    @Test
+    public void testNextBytes() {
+        Assert.assertArrayEquals(new byte[] { 0 }, BytesUtil.nextBytes(new byte[] {}));
+        Assert.assertArrayEquals(new byte[] { 1, 2, 0 }, BytesUtil.nextBytes(new byte[] { 1, 2 }));
+    }
+
+    @Test
+    public void testCompare() {
+        byte[] array = new byte[] { 1, 2 };
+
+        Assert.assertEquals(0, BytesUtil.compare(array, array));
+        Assert.assertEquals(-2, BytesUtil.compare(new byte[] { 1, 2 }, new byte[] { 3, 4 }));
+        Assert.assertEquals(0, BytesUtil.compare(new byte[] { 3, 4 }, new byte[] { 3, 4 }));
+    }
+
+    @Test
+    public void testMax() {
+        byte[] array = new byte[] { 3, 4 };
+
+        Assert.assertArrayEquals(array, BytesUtil.max(array, array));
+        Assert.assertArrayEquals(array, BytesUtil.max(new byte[] { 1, 2 }, array));
+    }
+
+    @Test
+    public void testMin() {
+        byte[] array = new byte[] { 1, 2 };
+        Assert.assertArrayEquals(array, BytesUtil.min(array, array));
+        Assert.assertArrayEquals(array, BytesUtil.min(array, new byte[] { 3, 4 }));
+    }
+
+    @SuppressWarnings("ConstantConditions")
+    @Test
+    public void testToHex() {
+        Assert.assertNull(BytesUtil.toHex(null));
+
+        Assert.assertEquals("0102", BytesUtil.toHex(new byte[] { 1, 2 }));
+    }
+
+    @Test
+    public void testHexStringToByteArray() {
+        Assert.assertNull(BytesUtil.hexStringToByteArray(null));
+
+        Assert.assertArrayEquals(new byte[] { -17, -5 }, BytesUtil.hexStringToByteArray("foob"));
+    }
+
+    @Test
+    public void toUtf8BytesTest() {
+        for (int i = 0; i < 100000; i++) {
+            String in = UUID.randomUUID().toString();
+            assertArrayEquals(Utils.getBytes(in), BytesUtil.writeUtf8(in));
+        }
+    }
+
+    @Test
+    public void toUtf8StringTest() {
+        for (int i = 0; i < 100000; i++) {
+            String str = UUID.randomUUID().toString();
+            byte[] in = Utils.getBytes(str);
+            assertEquals(new String(in, StandardCharsets.UTF_8), BytesUtil.readUtf8(in));
+        }
+    }
+
+    @Test
+    public void hexTest() {
+        final String text = "Somebody save your soul cause you've been sinning in this city I know";
+        final String hexString = BytesUtil.toHex(text.getBytes());
+        System.out.println(hexString);
+        final byte[] bytes = BytesUtil.hexStringToByteArray(hexString);
+        assertEquals(text, new String(bytes));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CountDownEventTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CountDownEventTest.java
new file mode 100644
index 0000000..2ee8a65
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CountDownEventTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CountDownEventTest {
+    @Test
+    public void testAwait() throws Exception {
+        CountDownEvent e = new CountDownEvent();
+        e.incrementAndGet();
+        e.incrementAndGet();
+        AtomicLong cost = new AtomicLong(0);
+        CountDownLatch latch = new CountDownLatch(1);
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                try {
+                    long start = System.currentTimeMillis();
+                    e.await();
+                    cost.set(System.currentTimeMillis() - start);
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+                latch.countDown();
+            }
+        });
+        Thread.sleep(1000);
+        e.countDown();
+        Thread.sleep(1000);
+        e.countDown();
+        latch.await();
+        assertEquals(2000, cost.get(), 50);
+    }
+
+    @Test(expected = InterruptedException.class)
+    public void testInterrupt() throws Exception {
+        CountDownEvent e = new CountDownEvent();
+        e.incrementAndGet();
+        e.incrementAndGet();
+        Thread thread = Thread.currentThread();
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                try {
+                    Thread.sleep(100);
+                    thread.interrupt();
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+
+            }
+        });
+        e.await();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CrcUtilTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CrcUtilTest.java
new file mode 100644
index 0000000..7d0776a
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/CrcUtilTest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class CrcUtilTest {
+
+    @Test
+    public void testCrc64() {
+        byte[] bs = "hello world".getBytes();
+        long c = CrcUtil.crc64(bs);
+        assertEquals(c, CrcUtil.crc64(bs));
+        assertEquals(c, CrcUtil.crc64(bs));
+        assertEquals(c, CrcUtil.crc64(bs, 0, bs.length));
+
+        ByteBuffer buf = ByteBuffer.wrap(bs);
+        assertEquals(c, CrcUtil.crc64(buf));
+
+        buf = ByteBuffer.allocateDirect(bs.length);
+        buf.put(bs);
+        buf.flip();
+        assertEquals(c, CrcUtil.crc64(buf));
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/EndpointTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/EndpointTest.java
new file mode 100644
index 0000000..4072624
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/EndpointTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class EndpointTest {
+
+    @Test
+    public void testToStringReset() {
+        final Endpoint ep = new Endpoint("192.168.1.1", 8080);
+        assertEquals("192.168.1.1:8080", ep.toString());
+        assertEquals("192.168.1.1:8080", ep.toString());
+    }
+
+    @Test
+    public void testCopy() {
+        final Endpoint ep = new Endpoint("192.168.1.1", 8080);
+        assertEquals("192.168.1.1:8080", ep.toString());
+        assertEquals("192.168.1.1:8080", ep.toString());
+        assertEquals(ep, ep.copy());
+    }
+
+    @Test
+    public void testEqualsHashCode() {
+        final Endpoint ep1 = new Endpoint("192.168.1.1", 8080);
+        final Endpoint ep2 = new Endpoint("192.168.1.1", 8080);
+        assertEquals(ep1, ep2);
+        assertEquals(ep1.hashCode(), ep2.hashCode());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/FileOutputSignalHandlerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/FileOutputSignalHandlerTest.java
new file mode 100644
index 0000000..34479fa
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/FileOutputSignalHandlerTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author jiachun.fjc
+ */
+public class FileOutputSignalHandlerTest {
+
+    @Test
+    public void testGetOutputFileWithEmptyPath() throws IOException {
+        final File f = getOutputFile("", "test1.log");
+        assertTrue(f.exists());
+        FileUtils.forceDelete(f);
+    }
+
+    @Test
+    public void testGetOutputFileWithPath() throws IOException {
+        final String path = "abc";
+        final File f = getOutputFile(path, "test2.log");
+        assertTrue(f.exists());
+        FileUtils.forceDelete(new File(path));
+    }
+
+    @Test
+    public void testGetOutputFileWithAbsolutePath() throws IOException {
+        final String path = Paths.get("cde").toAbsolutePath().toString();
+        final File f = getOutputFile(path, "test3.log");
+        assertTrue(f.exists());
+        FileUtils.forceDelete(new File(path));
+    }
+
+    private File getOutputFile(final String path, final String baseName) throws IOException {
+        return new FileOutputSignalHandler() {
+            @Override
+            public void handle(String signalName) {
+            }
+        }.getOutputFile(path, baseName);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/JRaftServiceLoaderTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/JRaftServiceLoaderTest.java
new file mode 100644
index 0000000..2253280
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/JRaftServiceLoaderTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.List;
+import java.util.ServiceConfigurationError;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class JRaftServiceLoaderTest {
+
+    @Test
+    public void serviceNotFoundTest() {
+        try {
+            JRaftServiceLoader.load(NoImplTest.class).first();
+            fail("fail");
+        } catch (final ServiceConfigurationError e) {
+            assertEquals(
+                "com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$NoImplTest: could not find any implementation for class",
+                e.getMessage());
+        }
+    }
+
+    @Test
+    public void serviceSortTest() {
+        final List serviceList = JRaftServiceLoader.load(SortTest.class) //
+            .sort();
+        assertEquals(3, serviceList.size());
+        assertEquals(SortImpl9Test.class, serviceList.get(0).getClass());
+        assertEquals(SortImpl2Test.class, serviceList.get(1).getClass());
+        assertEquals(SortImpl1Test.class, serviceList.get(2).getClass());
+    }
+
+    @Test
+    public void serviceGetFirstTest() {
+        final SortTest service = JRaftServiceLoader.load(SortTest.class) //
+            .first();
+        assertNotNull(service);
+        assertEquals(SortImpl9Test.class, service.getClass());
+    }
+
+    @Test
+    public void serviceFindTest() {
+        final SortTest service = JRaftServiceLoader.load(SortTest.class) //
+            .find("sort2");
+        assertNotNull(service);
+        assertEquals(SortImpl2Test.class, service.getClass());
+    }
+
+    public interface NoImplTest {
+    }
+
+    public interface SortTest {
+    }
+
+    @SPI(name = "sort1", priority = 1)
+    public static class SortImpl1Test implements SortTest {
+    }
+
+    @SPI(name = "sort2", priority = 2)
+    public static class SortImpl2Test implements SortTest {
+    }
+
+    @SPI(name = "sort9", priority = 9)
+    public static class SortImpl9Test implements SortTest {
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclableByteBufferListTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclableByteBufferListTest.java
new file mode 100644
index 0000000..26130e1
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclableByteBufferListTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertSame;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RecyclableByteBufferListTest {
+
+    @Test(expected = IllegalStateException.class)
+    public void testMultipleRecycle() {
+        final RecyclableByteBufferList object = RecyclableByteBufferList.newInstance();
+        object.recycle();
+        object.recycle();
+    }
+
+    @Test
+    public void testMultipleRecycleAtDifferentThread() throws InterruptedException {
+        final RecyclableByteBufferList object = RecyclableByteBufferList.newInstance();
+        final Thread thread1 = new Thread(object::recycle);
+        thread1.start();
+        thread1.join();
+        assertSame(object, RecyclableByteBufferList.newInstance());
+    }
+
+    @Test
+    public void testRecycle() {
+        final RecyclableByteBufferList object = RecyclableByteBufferList.newInstance();
+        object.recycle();
+        final RecyclableByteBufferList object2 = RecyclableByteBufferList.newInstance();
+        Assert.assertSame(object, object2);
+        object2.recycle();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclersTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclersTest.java
new file mode 100644
index 0000000..2c44fe2
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RecyclersTest.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RecyclersTest {
+
+    private static Recyclers newRecyclers(final int max) {
+        return new Recyclers(max) {
+
+            @Override
+            protected RecyclableObject newObject(final Recyclers.Handle handle) {
+                return new RecyclableObject(handle);
+            }
+        };
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testMultipleRecycle() {
+        final Recyclers recyclers = newRecyclers(16);
+        final RecyclableObject object = recyclers.get();
+        recyclers.recycle(object, object.handle);
+        recyclers.recycle(object, object.handle);
+    }
+
+    @Test
+    public void testMultipleRecycleAtDifferentThread() throws InterruptedException {
+        final Recyclers recyclers = newRecyclers(512);
+        final RecyclableObject object = recyclers.get();
+        final Thread thread1 = new Thread(() -> recyclers.recycle(object, object.handle));
+        thread1.start();
+        thread1.join();
+        assertSame(object, recyclers.get());
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testRecycleMoreThanOnceAtDifferentThread() throws InterruptedException {
+        final Recyclers recyclers = newRecyclers(1024);
+        final RecyclableObject object = recyclers.get();
+
+        final AtomicReference exceptionStore = new AtomicReference<>();
+        final Thread thread1 = new Thread(() -> recyclers.recycle(object, object.handle));
+        thread1.start();
+        thread1.join();
+
+        final Thread thread2 = new Thread(() -> {
+            try {
+                recyclers.recycle(object, object.handle);
+            } catch (IllegalStateException e) {
+                exceptionStore.set(e);
+            }
+        });
+        thread2.start();
+        thread2.join();
+        IllegalStateException exception = exceptionStore.get();
+        if (exception != null) {
+            throw exception;
+        }
+    }
+
+    @Test
+    public void testRecycle() {
+        final Recyclers recyclers = newRecyclers(16);
+        final RecyclableObject object = recyclers.get();
+        recyclers.recycle(object, object.handle);
+        final RecyclableObject object2 = recyclers.get();
+        Assert.assertSame(object, object2);
+        recyclers.recycle(object2, object2.handle);
+    }
+
+    @Test
+    public void testRecycleDisable() {
+        final Recyclers recyclers = newRecyclers(-1);
+        final RecyclableObject object = recyclers.get();
+        recyclers.recycle(object, object.handle);
+        final RecyclableObject object2 = recyclers.get();
+        assertNotSame(object, object2);
+        recyclers.recycle(object2, object2.handle);
+    }
+
+    @Test
+    public void testMaxCapacity() {
+        testMaxCapacity(300);
+        Random rand = new Random();
+        for (int i = 0; i < 50; i++) {
+            testMaxCapacity(rand.nextInt(1000) + 256); // 256 - 1256
+        }
+    }
+
+    private static void testMaxCapacity(final int maxCapacity) {
+        final Recyclers recyclers = newRecyclers(maxCapacity);
+        final RecyclableObject[] objects = new RecyclableObject[maxCapacity * 3];
+        for (int i = 0; i < objects.length; i++) {
+            objects[i] = recyclers.get();
+        }
+
+        for (int i = 0; i < objects.length; i++) {
+            recyclers.recycle(objects[i], objects[i].handle);
+            objects[i] = null;
+        }
+
+        assertTrue("The threadLocalCapacity (" + recyclers.threadLocalCapacity() + ") must be <= maxCapacity ("
+                   + maxCapacity + ") as we not pool all new handles internally",
+            maxCapacity >= recyclers.threadLocalCapacity());
+    }
+
+    static final class RecyclableObject {
+
+        private final Recyclers.Handle handle;
+
+        private RecyclableObject(Recyclers.Handle handle) {
+            this.handle = handle;
+        }
+
+        public Recyclers.Handle getHandle() {
+            return handle;
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RepeatedTimerTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RepeatedTimerTest.java
new file mode 100644
index 0000000..9aa6136
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/RepeatedTimerTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class RepeatedTimerTest {
+    private static class TestTimer extends RepeatedTimer {
+        final AtomicInteger counter     = new AtomicInteger(0);
+        final AtomicInteger destroyed   = new AtomicInteger(0);
+        volatile int        nextTimeout = -1;
+
+        public TestTimer(String name, int timeoutMs) {
+            super(name, timeoutMs);
+        }
+
+        @Override
+        protected int adjustTimeout(final int timeoutMs) {
+            if (nextTimeout > 0) {
+                return nextTimeout;
+            } else {
+                return timeoutMs;
+            }
+        }
+
+        @Override
+        protected void onDestroy() {
+            destroyed.incrementAndGet();
+        }
+
+        @Override
+        protected void onTrigger() {
+            counter.incrementAndGet();
+        }
+
+    }
+
+    private TestTimer timer;
+
+    @Before
+    public void setup() {
+        this.timer = new TestTimer("test", 50);
+    }
+
+    @After
+    public void teardown() {
+        this.timer.destroy();
+    }
+
+    @Test
+    public void testStartTrigger() throws Exception {
+        assertEquals(0, this.timer.counter.get());
+        this.timer.start();
+        Thread.sleep(1000);
+        assertEquals(20, this.timer.counter.get(), 3);
+    }
+
+    @Test
+    public void testStopStart() throws Exception {
+        assertEquals(0, this.timer.counter.get());
+        this.timer.start();
+        Thread.sleep(1000);
+        assertEquals(20, this.timer.counter.get(), 5);
+        this.timer.stop();
+        Thread.sleep(1000);
+        assertEquals(20, this.timer.counter.get(), 5);
+        this.timer.start();
+        Thread.sleep(1000);
+        assertEquals(40, this.timer.counter.get(), 5);
+    }
+
+    @Test
+    public void testRunOnce() throws Exception {
+        assertEquals(0, this.timer.counter.get());
+        this.timer.start();
+        this.timer.runOnceNow();
+        assertEquals(1, this.timer.counter.get());
+        Thread.sleep(1000);
+        assertEquals(20, this.timer.counter.get(), 3);
+    }
+
+    @Test
+    public void testDestroy() throws Exception {
+        this.timer.start();
+        assertEquals(0, this.timer.destroyed.get());
+        Thread.sleep(100);
+        this.timer.destroy();
+        assertEquals(1, this.timer.destroyed.get());
+    }
+
+    @Test
+    public void testAdjustTimeout() throws Exception {
+        this.timer.nextTimeout = 100;
+        this.timer.start();
+        Thread.sleep(1000);
+        assertEquals(10, this.timer.counter.get(), 3);
+    }
+
+    @Test
+    public void testReset() throws Exception {
+        this.timer.start();
+        assertEquals(50, this.timer.getTimeoutMs());
+        for (int i = 0; i < 10; i++) {
+            Thread.sleep(80);
+            this.timer.reset();
+        }
+        assertEquals(10, this.timer.counter.get(), 3);
+        this.timer.reset(100);
+        for (int i = 0; i < 10; i++) {
+            Thread.sleep(80);
+            this.timer.reset();
+        }
+        assertEquals(10, this.timer.counter.get(), 3);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SegmentListTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SegmentListTest.java
new file mode 100644
index 0000000..ce9efe2
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SegmentListTest.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import io.netty.util.internal.ThreadLocalRandom;
+
+public class SegmentListTest {
+
+    private SegmentList list;
+
+    @Before
+    public void setup() {
+        this.list = new SegmentList<>(true);
+    }
+
+    @Test
+    public void testAddGet() {
+        assertTrue(this.list.isEmpty());
+        fillList();
+        assertFilledList();
+        System.out.println(this.list);
+    }
+
+    private void assertFilledList() {
+        for (int i = 0; i < 1000; i++) {
+            assertEquals(i, (int) this.list.get(i));
+        }
+        assertEquals(1000, this.list.size());
+        assertFalse(this.list.isEmpty());
+        assertEquals(1000 / SegmentList.SEGMENT_SIZE + 1, this.list.segmentSize());
+    }
+
+    private void fillList() {
+        int originSize = this.list.size();
+        for (int i = 0; i < 1000; i++) {
+            this.list.add(i);
+            assertEquals(originSize + i + 1, this.list.size());
+        }
+    }
+
+    @Test
+    public void testAddAllGet() {
+        List tmpList = new ArrayList<>();
+        for (int i = 0; i < 1000; i++) {
+            tmpList.add(i);
+        }
+
+        this.list.addAll(tmpList);
+        assertFilledList();
+
+        this.list.removeFromFirstWhen(x -> x < 100);
+        assertEquals(900, this.list.size());
+
+        this.list.addAll(tmpList);
+        assertEquals(1900, this.list.size());
+
+        for (int i = 0; i < 1900; i++) {
+            if (i < 900) {
+                assertEquals(i + 100, (int) this.list.get(i));
+            } else {
+                assertEquals(i - 900, (int) this.list.get(i));
+            }
+        }
+
+    }
+
+    @Test
+    public void testRemoveFromFirst() {
+        fillList();
+
+        int len = SegmentList.SEGMENT_SIZE - 1;
+        this.list.removeFromFirst(len);
+
+        assertEquals(1000 - len, this.list.size());
+
+        for (int i = 0; i < 1000 - len; i++) {
+            assertEquals(i + len, (int) this.list.get(i));
+        }
+
+        this.list.removeFromFirst(100);
+        assertEquals(1000 - len - 100, this.list.size());
+
+        for (int i = 0; i < 1000 - len - 100; i++) {
+            assertEquals(i + len + 100, (int) this.list.get(i));
+        }
+
+        this.list.removeFromFirst(1000 - len - 100);
+        assertTrue(this.list.isEmpty());
+        assertEquals(0, this.list.segmentSize());
+        assertNull(this.list.peekFirst());
+        assertNull(this.list.peekLast());
+    }
+
+    @Test
+    public void testRemoveFromFirstWhen() {
+        fillList();
+        this.list.removeFromFirstWhen(x -> x < 200);
+        assertEquals(800, this.list.size());
+        assertEquals(200, (int) this.list.get(0));
+
+        for (int i = 0; i < 800; i++) {
+            assertEquals(200 + i, (int) this.list.get(i));
+        }
+
+        this.list.removeFromFirstWhen(x -> x < 500);
+        assertEquals(500, this.list.size());
+        for (int i = 0; i < 500; i++) {
+            assertEquals(500 + i, (int) this.list.get(i));
+        }
+
+        this.list.removeFromFirstWhen(x -> x < 1000);
+        assertTrue(this.list.isEmpty());
+        assertEquals(0, this.list.segmentSize());
+
+        fillList();
+        assertFilledList();
+    }
+
+    @Test
+    public void testRemoveFromLastWhen() {
+        fillList();
+
+        // remove elements is greater or equal to 150.
+        this.list.removeFromLastWhen(x -> x >= 150);
+        assertEquals(150, this.list.size());
+        assertFalse(this.list.isEmpty());
+        for (int i = 0; i < 150; i++) {
+            assertEquals(i, (int) this.list.get(i));
+        }
+        try {
+            this.list.get(151);
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+
+        }
+        assertEquals(150 / SegmentList.SEGMENT_SIZE + 1, this.list.segmentSize());
+
+        // remove  elements is greater or equal to 32.
+        this.list.removeFromLastWhen(x -> x >= 32);
+        assertEquals(32, this.list.size());
+        assertFalse(this.list.isEmpty());
+        for (int i = 0; i < 32; i++) {
+            assertEquals(i, (int) this.list.get(i));
+        }
+        try {
+            this.list.get(32);
+            fail();
+        } catch (IndexOutOfBoundsException e) {
+
+        }
+        assertEquals(1, this.list.segmentSize());
+
+        // Add elements again.
+        fillList();
+        assertEquals(1032, this.list.size());
+        for (int i = 0; i < 1032; i++) {
+            if (i < 32) {
+                assertEquals(i, (int) this.list.get(i));
+            } else {
+                assertEquals(i - 32, (int) this.list.get(i));
+            }
+        }
+    }
+
+    @Test
+    public void testAddPeek() {
+        for (int i = 0; i < 1000; i++) {
+            this.list.add(i);
+            assertEquals(i, (int) this.list.peekLast());
+            assertEquals(0, (int) this.list.peekFirst());
+        }
+    }
+
+    @Test
+    public void simpleBenchmark() {
+        int warmupRepeats = 10_0000;
+        int repeats = 100_0000;
+
+        double arrayDequeOps = 0;
+        double segListOps = 0;
+        // test ArrayDequeue
+        {
+            ArrayDeque deque = new ArrayDeque<>();
+            System.gc();
+            // wramup
+            benchArrayDequeue(warmupRepeats, deque);
+            deque.clear();
+            System.gc();
+            long startNs = System.nanoTime();
+            benchArrayDequeue(repeats, deque);
+            long costMs = (System.nanoTime() - startNs) / repeats;
+            arrayDequeOps = repeats * 3.0 / costMs * 1000;
+            System.out.println("ArrayDeque, cost:" + costMs + ", ops: " + arrayDequeOps);
+        }
+        // test SegmentList
+        {
+            System.gc();
+            // warmup
+            benchSegmentList(warmupRepeats);
+
+            this.list.clear();
+            System.gc();
+
+            long startNs = System.nanoTime();
+            benchSegmentList(repeats);
+            long costMs = (System.nanoTime() - startNs) / repeats;
+            segListOps = repeats * 3.0 / costMs * 1000;
+            System.out.println("SegmentList, cost:" + costMs + ", ops: " + segListOps);
+            this.list.clear();
+        }
+
+        System.out.println("Improvement:" + Math.round((segListOps - arrayDequeOps) / arrayDequeOps * 100) + "%");
+
+    }
+
+    private void benchArrayDequeue(final int repeats, final ArrayDeque deque) {
+        int start = 0;
+        for (int i = 0; i < repeats; i++) {
+            List tmpList = genData(start);
+            deque.addAll(tmpList);
+            //            for (Integer o : tmpList) {
+            //                deque.add(o);
+            //            }
+            int removePos = start + ThreadLocalRandom.current().nextInt(tmpList.size());
+
+            deque.get(removePos - start);
+
+            int index = 0;
+            for (int j = 0; j < deque.size(); j++) {
+                if (deque.get(j) > removePos) {
+                    index = j;
+                    break;
+                }
+            }
+
+            if (index > 0) {
+                deque.removeRange(0, index);
+            }
+
+            start += tmpList.size();
+        }
+    }
+
+    private List genData(final int start) {
+        int n = ThreadLocalRandom.current().nextInt(500) + 10;
+        List tmpList = new ArrayList<>(n);
+        for (int i = 0; i < n; i++) {
+            tmpList.add(i + start);
+        }
+        return tmpList;
+    }
+
+    private void benchSegmentList(final int repeats) {
+        int start = 0;
+
+        for (int i = 0; i < repeats; i++) {
+            List tmpList = genData(start);
+            this.list.addAll(tmpList);
+            //            for(Integer o: tmpList) {
+            //                list.add(o);
+            //            }
+            int removePos = start + ThreadLocalRandom.current().nextInt(tmpList.size());
+            this.list.get(removePos - start);
+            this.list.removeFromFirstWhen(x -> x <= removePos);
+            start += tmpList.size();
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SignalHelperTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SignalHelperTest.java
new file mode 100644
index 0000000..079952c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/SignalHelperTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class SignalHelperTest {
+
+    public static void main(String[] args) throws InterruptedException {
+        // test with:
+        //
+        // kill -s USR2 pid
+
+        final List handlers = new ArrayList<>();
+        handlers.add((signalName) -> System.out.println("signal test: " + signalName));
+
+        if (SignalHelper.supportSignal()) {
+            SignalHelper.addSignal(SignalHelper.SIG_USR2, handlers);
+        }
+
+        Thread.sleep(300000);
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ThreadIdTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ThreadIdTest.java
new file mode 100644
index 0000000..3d4bc4d
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/ThreadIdTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class ThreadIdTest implements ThreadId.OnError {
+    private ThreadId     id;
+    private volatile int errorCode = -1;
+
+    @Override
+    public void onError(final ThreadId id, final Object data, final int errorCode) {
+        assertSame(id, this.id);
+        this.errorCode = errorCode;
+    }
+
+    @Before
+    public void setup() {
+        this.id = new ThreadId(this, this);
+    }
+
+    @Test
+    public void testLockUnlock() throws Exception {
+        assertSame(this, this.id.lock());
+        AtomicLong cost = new AtomicLong(0);
+        CountDownLatch latch = new CountDownLatch(1);
+        new Thread() {
+            @Override
+            public void run() {
+                long start = System.currentTimeMillis();
+                ThreadIdTest.this.id.lock();
+                cost.set(System.currentTimeMillis() - start);
+                latch.countDown();
+            }
+        }.start();
+        Thread.sleep(1000);
+        this.id.unlock();
+        latch.await();
+        assertEquals(1000, cost.get(), 10);
+    }
+
+    @Test
+    public void testSetError() throws Exception {
+        this.id.setError(100);
+        assertEquals(100, this.errorCode);
+        CountDownLatch latch = new CountDownLatch(1);
+        new Thread() {
+            @Override
+            public void run() {
+                ThreadIdTest.this.id.setError(99);
+                latch.countDown();
+            }
+        }.start();
+        latch.await();
+        assertEquals(99, this.errorCode);
+    }
+
+    @Test
+    public void testUnlockAndDestroy() throws Exception {
+        AtomicInteger lockSuccess = new AtomicInteger(0);
+        CountDownLatch latch = new CountDownLatch(10);
+        this.id.lock();
+        for (int i = 0; i < 10; i++) {
+            new Thread() {
+                @Override
+                public void run() {
+                    if (ThreadIdTest.this.id.lock() != null) {
+                        lockSuccess.incrementAndGet();
+                    }
+                    latch.countDown();
+                }
+            }.start();
+        }
+        this.id.unlockAndDestroy();
+        latch.await();
+        assertEquals(0, lockSuccess.get());
+        assertNull(this.id.lock());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/Utf8CodecBenchmark.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/Utf8CodecBenchmark.java
new file mode 100644
index 0000000..ed6d6eb
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/Utf8CodecBenchmark.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.charset.StandardCharsets;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class Utf8CodecBenchmark {
+    /**
+     Benchmark                                Mode  Cnt      Score       Error   Units
+     Utf8CodecBenchmark.defaultToUtf8Bytes   thrpt    3  13744.773 ±  2188.618  ops/ms
+     Utf8CodecBenchmark.defaultToUtf8String  thrpt    3  18136.042 ± 10964.592  ops/ms
+     Utf8CodecBenchmark.unsafeToUtf8Bytes    thrpt    3  21743.863 ±   228.019  ops/ms
+     Utf8CodecBenchmark.unsafeToUtf8String   thrpt    3  20670.839 ±  9921.726  ops/ms
+     */
+
+    private String str;
+    private byte[] bytes;
+
+    @Setup
+    public void setup() {
+        str = UUID.randomUUID().toString();
+        bytes = Utils.getBytes(str);
+    }
+
+    @SuppressWarnings("all")
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void defaultToUtf8Bytes() {
+        Utils.getBytes(str);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void unsafeToUtf8Bytes() {
+        BytesUtil.writeUtf8(str);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void defaultToUtf8String() {
+        new String(bytes, StandardCharsets.UTF_8);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.Throughput)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void unsafeToUtf8String() {
+        BytesUtil.readUtf8(bytes);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(Utf8CodecBenchmark.class.getSimpleName()) //
+            .warmupIterations(3) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/UtilsTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/UtilsTest.java
new file mode 100644
index 0000000..9b77eaf
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/UtilsTest.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * 
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-10 5:51:20 PM
+ */
+public class UtilsTest {
+
+    @Test
+    public void testRunThread() throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        Utils.runInThread(new Runnable() {
+
+            @Override
+            public void run() {
+                latch.countDown();
+            }
+        });
+        latch.await();
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void tetsVerifyGroupId1() {
+        Utils.verifyGroupId("");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void tetsVerifyGroupId2() {
+        Utils.verifyGroupId(null);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void tetsVerifyGroupId3() {
+        Utils.verifyGroupId("1abc");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void tetsVerifyGroupId4() {
+        Utils.verifyGroupId("*test");
+    }
+
+    @Test
+    public void tetsVerifyGroupId5() {
+        Utils.verifyGroupId("t");
+        Utils.verifyGroupId("T");
+        Utils.verifyGroupId("Test");
+        Utils.verifyGroupId("test");
+        Utils.verifyGroupId("test-hello");
+        Utils.verifyGroupId("test123");
+        Utils.verifyGroupId("t_hello");
+    }
+
+    @Test
+    public void testRunClosure() throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        Utils.runClosureInThread(new Closure() {
+
+            @Override
+            public void run(Status status) {
+                assertTrue(status.isOk());
+                latch.countDown();
+            }
+        });
+        latch.await();
+    }
+
+    @Test
+    public void testRunClosureWithStatus() throws Exception {
+        CountDownLatch latch = new CountDownLatch(1);
+        Utils.runClosureInThread(new Closure() {
+
+            @Override
+            public void run(Status status) {
+                assertFalse(status.isOk());
+                Assert.assertEquals(RaftError.EACCES.getNumber(), status.getCode());
+                assertEquals("test 99", status.getErrorMsg());
+                latch.countDown();
+            }
+        }, new Status(RaftError.EACCES, "test %d", 99));
+        latch.await();
+    }
+
+    @Test
+    public void test_getProcessId() {
+        long pid = Utils.getProcessId(-1);
+        assertNotEquals(-1, pid);
+        System.out.println("test pid:" + pid);
+    }
+
+    @Test
+    public void testAllocateExpandByteBuffer() {
+        ByteBuffer buf = Utils.allocate(128);
+        assertEquals(0, buf.position());
+        assertEquals(128, buf.capacity());
+        assertEquals(128, buf.remaining());
+
+        buf.put("hello".getBytes());
+        assertEquals(5, buf.position());
+
+        buf = Utils.expandByteBufferAtLeast(buf, 128);
+        assertEquals(5, buf.position());
+        assertEquals(1152, buf.capacity());
+        assertEquals(1147, buf.remaining());
+
+        buf = Utils.expandByteBufferAtLeast(buf, 2048);
+        assertEquals(5, buf.position());
+        assertEquals(1152 + 2048, buf.capacity());
+        assertEquals(1147 + 2048, buf.remaining());
+    }
+
+    @Test
+    public void testParsePeerId() {
+        String pid = "192.168.1.88:5566";
+        String[] result = Utils.parsePeerId(pid);
+        String[] expecteds = { "192.168.1.88", "5566" };
+        Assert.assertTrue(result.length == 2);
+        Assert.assertArrayEquals(expecteds, result);
+
+        pid = "[fe80:0:0:0:6450:aa3c:cd98:ed0f]:8847";
+        result = Utils.parsePeerId(pid);
+        expecteds = new String[] { "[fe80:0:0:0:6450:aa3c:cd98:ed0f]", "8847" };
+        Assert.assertTrue(result.length == 2);
+        Assert.assertArrayEquals(expecteds, result);
+
+        pid = "192.168.1.88:5566:9";
+        result = Utils.parsePeerId(pid);
+        expecteds = new String[] { "192.168.1.88", "5566", "9" };
+        Assert.assertTrue(result.length == 3);
+        Assert.assertArrayEquals(expecteds, result);
+
+        pid = "[fe80:0:0:0:6450:aa3c:cd98:ed0f]:8847:9";
+        result = Utils.parsePeerId(pid);
+        expecteds = new String[] { "[fe80:0:0:0:6450:aa3c:cd98:ed0f]", "8847", "9" };
+        Assert.assertTrue(result.length == 3);
+        Assert.assertArrayEquals(expecteds, result);
+
+        pid = "192.168.1.88:5566:0:6";
+        result = Utils.parsePeerId(pid);
+        expecteds = new String[] { "192.168.1.88", "5566", "0", "6" };
+        Assert.assertTrue(result.length == 4);
+        Assert.assertArrayEquals(expecteds, result);
+
+        pid = "[fe80:0:0:0:6450:aa3c:cd98:ed0f]:8847:0:6";
+        result = Utils.parsePeerId(pid);
+        expecteds = new String[] { "[fe80:0:0:0:6450:aa3c:cd98:ed0f]", "8847", "0", "6" };
+        Assert.assertTrue(result.length == 4);
+        Assert.assertArrayEquals(expecteds, result);
+
+        boolean ex1 = false;
+        try {
+            pid = "[192.168.1].88:eee:x:b:j";
+            Utils.parsePeerId(pid);
+        } catch (Exception e) {
+            ex1 = true;
+        }
+        Assert.assertTrue(ex1);
+
+        boolean ex2 = false;
+        try {
+            pid = "[dsfsadf]:eee:x:b:j";
+            Utils.parsePeerId(pid);
+        } catch (Exception e) {
+            ex2 = true;
+        }
+        Assert.assertTrue(ex2);
+
+    }
+
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphoreTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphoreTest.java
new file mode 100644
index 0000000..31e627d
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/AdjustableSemaphoreTest.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class AdjustableSemaphoreTest {
+
+    @Test
+    public void updateMaxPermitsTest() {
+        final AdjustableSemaphore semaphore = new AdjustableSemaphore(5);
+        Assert.assertEquals(5, semaphore.availablePermits());
+        Assert.assertEquals(5, semaphore.getMaxPermits());
+        for (int i = 0; i < 5; i++) {
+            Assert.assertTrue(semaphore.tryAcquire());
+        }
+        Assert.assertFalse(semaphore.tryAcquire());
+        Assert.assertEquals(0, semaphore.availablePermits());
+        Assert.assertEquals(5, semaphore.getMaxPermits());
+        for (int i = 0; i < 5; i++) {
+            semaphore.release();
+        }
+        Assert.assertEquals(5, semaphore.availablePermits());
+
+        // decrease
+        semaphore.setMaxPermits(2);
+        Assert.assertEquals(2, semaphore.getMaxPermits());
+        Assert.assertEquals(2, semaphore.availablePermits());
+        for (int i = 0; i < 2; i++) {
+            Assert.assertTrue(semaphore.tryAcquire());
+        }
+        Assert.assertFalse(semaphore.tryAcquire());
+        Assert.assertEquals(0, semaphore.availablePermits());
+        Assert.assertEquals(2, semaphore.getMaxPermits());
+
+        for (int i = 0; i < 2; i++) {
+            semaphore.release();
+        }
+        // increase
+        semaphore.setMaxPermits(10);
+        Assert.assertEquals(10, semaphore.getMaxPermits());
+        Assert.assertEquals(10, semaphore.availablePermits());
+        for (int i = 0; i < 10; i++) {
+            Assert.assertTrue(semaphore.tryAcquire());
+        }
+        Assert.assertFalse(semaphore.tryAcquire());
+        Assert.assertEquals(0, semaphore.availablePermits());
+        Assert.assertEquals(10, semaphore.getMaxPermits());
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLockTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLockTest.java
new file mode 100644
index 0000000..c269f23
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/LongHeldDetectingReadWriteLockTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReadWriteLock;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class LongHeldDetectingReadWriteLockTest {
+
+    @Test
+    public void testLongHeldWriteLock() throws InterruptedException {
+        final ReadWriteLock readWriteLock = new LongHeldDetectingReadWriteLock(100, TimeUnit.MILLISECONDS) {
+
+            @Override
+            public void report(AcquireMode acquireMode, Thread owner, Collection queuedThreads, long blockedNanos) {
+                System.out.println("currentThread=" + Thread.currentThread() +
+                " acquireMode=" + acquireMode +
+                " lockOwner=" + owner +
+                " queuedThreads= " + queuedThreads +
+                " blockedMs=" + TimeUnit.NANOSECONDS.toMillis(blockedNanos));
+
+                Assert.assertTrue(Thread.currentThread().getName().contains("read-lock-thread"));
+                Assert.assertSame(AcquireMode.Read, acquireMode);
+                Assert.assertEquals("write-lock-thread", owner.getName());
+                Assert.assertEquals(2000, TimeUnit.NANOSECONDS.toMillis(blockedNanos), 100);
+            }
+        };
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            readWriteLock.writeLock().lock();
+            latch.countDown();
+            try {
+                Thread.sleep(2000);
+            } catch (final InterruptedException e) {
+                e.printStackTrace();
+            } finally {
+                readWriteLock.writeLock().unlock();
+            }
+        }, "write-lock-thread") //
+        .start();
+
+        latch.await();
+
+        final CountDownLatch latch1 = new CountDownLatch(2);
+        new Thread(() -> {
+            readWriteLock.readLock().lock();
+            readWriteLock.readLock().unlock();
+            latch1.countDown();
+        }, "read-lock-thread-1").start();
+
+        new Thread(() -> {
+            readWriteLock.readLock().lock();
+            readWriteLock.readLock().unlock();
+            latch1.countDown();
+        }, "read-lock-thread-2").start();
+
+        latch1.await();
+    }
+
+    @Test
+    public void testLongHeldReadLock() throws InterruptedException {
+        final ReadWriteLock readWriteLock = new LongHeldDetectingReadWriteLock(100, TimeUnit.MILLISECONDS) {
+
+            @Override
+            public void report(AcquireMode acquireMode, Thread owner, Collection queuedThreads, long blockedNanos) {
+                System.out.println("currentThread=" + Thread.currentThread() +
+                        " acquireMode=" + acquireMode +
+                        " lockOwner=" + owner +
+                        " queuedThreads= " + queuedThreads +
+                        " blockedMs=" + TimeUnit.NANOSECONDS.toMillis(blockedNanos));
+
+                Assert.assertTrue(Thread.currentThread().getName().contains("write-lock-thread"));
+                Assert.assertSame(AcquireMode.Write, acquireMode);
+                Assert.assertNull(owner);
+                Assert.assertEquals(2000, TimeUnit.NANOSECONDS.toMillis(blockedNanos), 100);
+            }
+        };
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            readWriteLock.readLock().lock();
+            latch.countDown();
+            try {
+                Thread.sleep(2000);
+            } catch (final InterruptedException e) {
+                e.printStackTrace();
+            } finally {
+                readWriteLock.readLock().unlock();
+            }
+        }, "read-lock-thread") //
+                .start();
+
+        latch.await();
+
+        final CountDownLatch latch1 = new CountDownLatch(2);
+        new Thread(() -> {
+            readWriteLock.writeLock().lock();
+            readWriteLock.writeLock().unlock();
+            latch1.countDown();
+        }, "write-lock-thread-1").start();
+
+        new Thread(() -> {
+            readWriteLock.writeLock().lock();
+            readWriteLock.writeLock().unlock();
+            latch1.countDown();
+        }, "write-lock-thread-2").start();
+
+        latch1.await();
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutorTest.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutorTest.java
new file mode 100644
index 0000000..923432c
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/MpscSingleThreadExecutorTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class MpscSingleThreadExecutorTest {
+
+    private static final ThreadFactory THREAD_FACTORY = new NamedThreadFactory("test", true);
+
+    @Test
+    public void testExecutorIsShutdownWithoutTask() {
+        final MpscSingleThreadExecutor executor = new MpscSingleThreadExecutor(1024, THREAD_FACTORY);
+
+        Assert.assertTrue(executor.shutdownGracefully());
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+        Assert.assertTrue(executor.isTerminated());
+    }
+
+    @Test
+    public void testExecutorIsShutdownWithTask() throws InterruptedException {
+        final MpscSingleThreadExecutor executor = new MpscSingleThreadExecutor(1024, THREAD_FACTORY);
+        final CountDownLatch latch = new CountDownLatch(10);
+        final AtomicLong ret = new AtomicLong(0);
+        for (int i = 0; i < 10; i++) {
+            executor.execute(() -> {
+                try {
+                    Thread.sleep(100);
+                    ret.incrementAndGet();
+                    latch.countDown();
+                } catch (final InterruptedException e) {
+                    e.printStackTrace();
+                }
+            });
+        }
+        Assert.assertTrue(executor.shutdownGracefully());
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+        Assert.assertTrue(executor.isTerminated());
+        latch.await();
+        Assert.assertEquals(10, ret.get());
+    }
+
+    @Test
+    public void testExecutorShutdownHooksWithoutTask() {
+        final MpscSingleThreadExecutor executor = new MpscSingleThreadExecutor(1024, THREAD_FACTORY);
+        final AtomicBoolean hookCalled = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(1);
+        executor.addShutdownHook(() -> {
+            hookCalled.set(true);
+            latch.countDown();
+        });
+        Assert.assertTrue(executor.shutdownGracefully());
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+        Assert.assertTrue(executor.isTerminated());
+        Assert.assertTrue(hookCalled.get());
+    }
+
+    @Test
+    public void testExecutorShutdownHooksWithTask() {
+        final MpscSingleThreadExecutor executor = new MpscSingleThreadExecutor(1024, THREAD_FACTORY);
+        final AtomicBoolean hookCalled = new AtomicBoolean(false);
+        final CountDownLatch latch = new CountDownLatch(11);
+        executor.addShutdownHook(() -> {
+            hookCalled.set(true);
+            latch.countDown();
+        });
+        final AtomicLong ret = new AtomicLong(0);
+        for (int i = 0; i < 10; i++) {
+            executor.execute(() -> {
+                try {
+                    Thread.sleep(100);
+                    ret.incrementAndGet();
+                    latch.countDown();
+                } catch (final InterruptedException e) {
+                    e.printStackTrace();
+                }
+            });
+        }
+        Assert.assertTrue(executor.shutdownGracefully());
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+        Assert.assertTrue(executor.isTerminated());
+        Assert.assertTrue(hookCalled.get());
+    }
+
+    @Test
+    public void testExecutorRejected() throws InterruptedException {
+        // 2048 is the minimum of maxPendingTasks
+        final int minMaxPendingTasks = 2048;
+        final MpscSingleThreadExecutor executor = new MpscSingleThreadExecutor(minMaxPendingTasks, THREAD_FACTORY);
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        final CountDownLatch latch2 = new CountDownLatch(1);
+
+        // add a block task
+        executor.execute(() -> {
+            try {
+                latch1.await();
+            } catch (final InterruptedException e) {
+                e.printStackTrace();
+            }
+            latch2.countDown();
+        });
+
+        // wait until the work is blocked
+        Thread.sleep(1000);
+
+        // fill the task queue
+        for (int i = 0; i < minMaxPendingTasks; i++) {
+            executor.execute(() -> {});
+        }
+
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+        executeShouldFail(executor);
+
+        latch1.countDown();
+        latch2.await();
+        executor.shutdownGracefully();
+    }
+
+    private static void executeShouldFail(final Executor executor) {
+        try {
+            executor.execute(() -> {
+                // Noop.
+            });
+            Assert.fail();
+        } catch (final RejectedExecutionException expected) {
+            // expected
+        }
+    }
+}
diff --git a/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.java b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.java
new file mode 100644
index 0000000..18cc0d8
--- /dev/null
+++ b/jraft-core/src/test/java/com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.util.concurrent;
+
+import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.LinkedTransferQueue;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.util.concurrent.DefaultEventExecutor;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+public class SingleThreadExecutorBenchmark {
+
+    private static final int TIMES   = 1000000;
+    private static final int THREADS = 32;
+
+    private ExecutorService  producers;
+
+    /*
+     * Benchmark                                                                         Mode  Cnt  Score   Error  Units
+     * SingleThreadExecutorBenchmark.defaultSingleThreadPollExecutor                    thrpt    3  1.266 ± 2.822  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutor                           thrpt    3  4.066 ± 4.990  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithConcurrentLinkedQueue  thrpt    3  3.470 ± 0.845  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedBlockingQueue    thrpt    3  2.643 ± 1.222  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedTransferQueue    thrpt    3  3.266 ± 1.613  ops/s
+     * SingleThreadExecutorBenchmark.nettyDefaultEventExecutor                          thrpt    3  2.290 ± 0.446  ops/s
+     *
+     * Benchmark                                                                         Mode  Cnt  Score   Error  Units
+     * SingleThreadExecutorBenchmark.defaultSingleThreadPollExecutor                    thrpt   10  1.389 ± 0.130  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutor                           thrpt   10  3.646 ± 0.323  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithConcurrentLinkedQueue  thrpt   10  3.386 ± 0.247  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedBlockingQueue    thrpt   10  2.535 ± 0.153  ops/s
+     * SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedTransferQueue    thrpt   10  3.184 ± 0.299  ops/s
+     * SingleThreadExecutorBenchmark.nettyDefaultEventExecutor                          thrpt   10  2.097 ± 0.075  ops/s
+     */
+
+    public static void main(String[] args) throws RunnerException {
+        final Options opt = new OptionsBuilder() //
+            .include(SingleThreadExecutorBenchmark.class.getSimpleName()) //
+            .warmupIterations(3) //
+            .measurementIterations(10) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+
+    @Setup
+    public void setup() {
+        this.producers = newProducers();
+    }
+
+    @TearDown
+    public void tearDown() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.producers);
+    }
+
+    @Benchmark
+    public void nettyDefaultEventExecutor() throws InterruptedException {
+        execute(new DefaultSingleThreadExecutor(
+            new DefaultEventExecutor(new NamedThreadFactory("netty_executor", true))));
+    }
+
+    @Benchmark
+    public void defaultSingleThreadPollExecutor() throws InterruptedException {
+        execute(new DefaultSingleThreadExecutor("default", TIMES));
+    }
+
+    @Benchmark
+    public void mpscSingleThreadExecutor() throws InterruptedException {
+        execute(new MpscSingleThreadExecutor(TIMES, new NamedThreadFactory("mpsc", true)));
+    }
+
+    @Benchmark
+    public void mpscSingleThreadExecutorWithConcurrentLinkedQueue() throws InterruptedException {
+        execute(new MpscSingleThreadExecutor(TIMES, new NamedThreadFactory("mpsc_clq", true)) {
+
+            @Override
+            protected Queue newTaskQueue(final int maxPendingTasks) {
+                return new ConcurrentLinkedQueue<>();
+            }
+        });
+    }
+
+    @Benchmark
+    public void mpscSingleThreadExecutorWithLinkedBlockingQueue() throws InterruptedException {
+        execute(new MpscSingleThreadExecutor(TIMES, new NamedThreadFactory("mpsc_lbq", true)) {
+
+            @Override
+            protected Queue newTaskQueue(final int maxPendingTasks) {
+                return new LinkedBlockingQueue<>(maxPendingTasks);
+            }
+        });
+    }
+
+    @Benchmark
+    public void mpscSingleThreadExecutorWithLinkedTransferQueue() throws InterruptedException {
+        execute(new MpscSingleThreadExecutor(TIMES, new NamedThreadFactory("mpsc_ltq", true)) {
+
+            @Override
+            protected Queue newTaskQueue(final int maxPendingTasks) {
+                return new LinkedTransferQueue<>();
+            }
+        });
+    }
+
+    private void execute(final SingleThreadExecutor executor) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(TIMES);
+        for (int i = 0; i < TIMES; i++) {
+            this.producers.execute(() -> executor.execute(latch::countDown));
+        }
+        latch.await();
+    }
+
+    private static ExecutorService newProducers() {
+        return ThreadPoolUtil.newBuilder() //
+            .coreThreads(THREADS) //
+            .maximumThreads(THREADS) //
+            .poolName("benchmark") //
+            .enableMetric(false) //
+            .workQueue(new ArrayBlockingQueue<>(TIMES)) //
+            .keepAliveSeconds(60L) //
+            .threadFactory(new NamedThreadFactory("benchmark", true)) //
+            .build();
+    }
+}
diff --git a/jraft-core/src/test/java/com/google/protobuf/ZeroByteStringHelperTest.java b/jraft-core/src/test/java/com/google/protobuf/ZeroByteStringHelperTest.java
new file mode 100644
index 0000000..cad0e5a
--- /dev/null
+++ b/jraft-core/src/test/java/com/google/protobuf/ZeroByteStringHelperTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.protobuf;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ZeroByteStringHelperTest {
+
+    @SuppressWarnings("ConstantConditions")
+    @Test
+    public void concatenateTest() {
+        final ThreadLocalRandom random = ThreadLocalRandom.current();
+        final List byteStrings = new ArrayList<>();
+        final List byteBuffers = new ArrayList<>();
+        final int segSize = 512;
+
+        int start;
+        int end = 0;
+        byte[] bytes;
+
+        for (int j = 0; j < 100; j++) {
+            start = end;
+            end = random.nextInt(start, start + segSize);
+            bytes = new byte[end];
+            for (int i = start; i < end; i++) {
+                bytes[i - start] = (byte) i;
+            }
+            byteBuffers.add(ByteBuffer.wrap(bytes));
+            byteStrings.add(ZeroByteStringHelper.wrap(bytes));
+        }
+        final ByteString rope = ZeroByteStringHelper.concatenate(byteBuffers);
+
+        int i = 0;
+        for (final ByteString bs : byteStrings) {
+            for (Byte b : bs) {
+                Assert.assertEquals(b.byteValue(), rope.byteAt(i++));
+            }
+        }
+        System.out.println(i);
+    }
+}
diff --git a/jraft-core/src/test/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest b/jraft-core/src/test/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest
new file mode 100644
index 0000000..413c621
--- /dev/null
+++ b/jraft-core/src/test/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest
@@ -0,0 +1,3 @@
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl1Test
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl2Test
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl9Test
\ No newline at end of file
diff --git a/jraft-core/src/test/resources/log4j2.xml b/jraft-core/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..d53e765
--- /dev/null
+++ b/jraft-core/src/test/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory
new file mode 100644
index 0000000..92d52a0
--- /dev/null
+++ b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.JRaftServiceFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.core.DefaultJRaftServiceFactory
\ No newline at end of file
diff --git a/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
new file mode 100644
index 0000000..8416bc1
--- /dev/null
+++ b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.rpc.impl.BoltRaftRpcFactory
\ No newline at end of file
diff --git a/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
new file mode 100644
index 0000000..c41f95b
--- /dev/null
+++ b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
@@ -0,0 +1,3 @@
+com.alipay.sofa.jraft.NodeDescribeSignalHandler
+com.alipay.sofa.jraft.NodeMetricsSignalHandler
+com.alipay.sofa.jraft.ThreadPoolMetricsSignalHandler
\ No newline at end of file
diff --git a/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory
new file mode 100644
index 0000000..943fda2
--- /dev/null
+++ b/jraft-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.timer.RaftTimerFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.util.timer.DefaultRaftTimerFactory
\ No newline at end of file
diff --git a/jraft-core/target/classes/cli.proto b/jraft-core/target/classes/cli.proto
new file mode 100644
index 0000000..93a464e
--- /dev/null
+++ b/jraft-core/target/classes/cli.proto
@@ -0,0 +1,107 @@
+syntax="proto2";
+
+package jraft;
+import "rpc.proto";
+
+option java_package="com.alipay.sofa.jraft.rpc";
+option java_outer_classname = "CliRequests";
+
+message AddPeerRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    required string peer_id = 3;
+}
+
+message AddPeerResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message RemovePeerRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    required string peer_id = 3;
+}
+
+message RemovePeerResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message ChangePeersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string new_peers = 3;
+}
+
+message ChangePeersResponse {
+    repeated string old_peers = 1;
+    repeated string new_peers = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message SnapshotRequest {
+    required string group_id = 1;
+    optional string peer_id = 2;
+};
+
+message ResetPeerRequest {
+    required string group_id = 1;
+    required string peer_id = 2;
+    repeated string old_peers = 3;
+    repeated string new_peers = 4;
+}
+
+message TransferLeaderRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    optional string peer_id = 3;
+}
+
+message GetLeaderRequest {
+    required string group_id = 1;
+    optional string peer_id = 2;
+}
+
+message GetLeaderResponse {
+    required string leader_id = 1;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message GetPeersRequest {
+    required string group_id = 1;
+    optional string leader_id = 2;
+    optional bool only_alive = 3 [default = false];
+}
+
+message GetPeersResponse {
+    repeated string peers = 1;
+    repeated string learners = 2;
+    optional ErrorResponse errorResponse = 99;
+}
+
+message AddLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message RemoveLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message ResetLearnersRequest {
+    required string group_id = 1;
+    required string leader_id = 2;
+    repeated string learners = 3;
+}
+
+message LearnersOpResponse {
+    repeated string old_learners = 1;
+    repeated string new_learners = 2;
+    optional ErrorResponse errorResponse = 99;
+}
diff --git a/jraft-core/target/classes/enum.proto b/jraft-core/target/classes/enum.proto
new file mode 100644
index 0000000..d8ab57b
--- /dev/null
+++ b/jraft-core/target/classes/enum.proto
@@ -0,0 +1,21 @@
+syntax="proto2";
+package jraft;
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "EnumOutter";
+
+enum EntryType {
+    ENTRY_TYPE_UNKNOWN = 0;
+    ENTRY_TYPE_NO_OP = 1;
+    ENTRY_TYPE_DATA = 2;
+    ENTRY_TYPE_CONFIGURATION= 3;
+};
+
+enum ErrorType {
+    ERROR_TYPE_NONE = 0;
+    ERROR_TYPE_LOG = 1;
+    ERROR_TYPE_STABLE = 2;
+    ERROR_TYPE_SNAPSHOT = 3;
+    ERROR_TYPE_STATE_MACHINE = 4;
+    ERROR_TYPE_META = 5;
+};
diff --git a/jraft-core/target/classes/gen.sh b/jraft-core/target/classes/gen.sh
new file mode 100644
index 0000000..d030056
--- /dev/null
+++ b/jraft-core/target/classes/gen.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+protoc -I=./  --descriptor_set_out=raft.desc --java_out=../java/ enum.proto local_file_meta.proto raft.proto local_storage.proto rpc.proto cli.proto log.proto
diff --git a/jraft-core/target/classes/local_file_meta.proto b/jraft-core/target/classes/local_file_meta.proto
new file mode 100644
index 0000000..c90a245
--- /dev/null
+++ b/jraft-core/target/classes/local_file_meta.proto
@@ -0,0 +1,18 @@
+syntax="proto2";
+
+package jraft;
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "LocalFileMetaOutter";
+
+
+enum FileSource {
+    FILE_SOURCE_LOCAL = 0;
+    FILE_SOURCE_REFERENCE = 1;
+}
+
+message LocalFileMeta {
+    optional bytes user_meta   = 1;
+    optional FileSource source = 2;
+    optional string checksum   = 3;
+}
diff --git a/jraft-core/target/classes/local_storage.proto b/jraft-core/target/classes/local_storage.proto
new file mode 100644
index 0000000..20ba7ff
--- /dev/null
+++ b/jraft-core/target/classes/local_storage.proto
@@ -0,0 +1,34 @@
+syntax="proto2";
+
+
+package jraft;
+
+import "raft.proto";
+import "local_file_meta.proto";
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "LocalStorageOutter";
+
+
+message ConfigurationPBMeta {
+    repeated string peers = 1;
+    repeated string old_peers = 2;
+};
+
+message LogPBMeta {
+    required int64 first_log_index = 1;
+};
+
+message StablePBMeta {
+    required int64 term = 1;
+    required string votedfor = 2;
+};
+
+message LocalSnapshotPbMeta {
+    message File {
+        required string name = 1;
+        optional LocalFileMeta meta = 2;
+    };
+    optional SnapshotMeta meta = 1;
+    repeated File files = 2;
+}
diff --git a/jraft-core/target/classes/log.proto b/jraft-core/target/classes/log.proto
new file mode 100644
index 0000000..b410fca
--- /dev/null
+++ b/jraft-core/target/classes/log.proto
@@ -0,0 +1,20 @@
+syntax="proto2";
+
+package jraft;
+
+import "enum.proto";
+
+option java_package="com.alipay.sofa.jraft.entity.codec.v2";
+option java_outer_classname = "LogOutter";
+
+message PBLogEntry {
+    required EntryType type = 1;
+    required int64 term = 2;
+    required int64 index = 3;
+    repeated bytes peers = 4;
+    repeated bytes old_peers = 5;
+    required bytes data = 6;
+    optional int64 checksum = 7;
+    repeated bytes learners = 8;
+    repeated bytes old_learners = 9;
+};
diff --git a/jraft-core/target/classes/raft.desc b/jraft-core/target/classes/raft.desc
new file mode 100644
index 0000000..6b0a544
Binary files /dev/null and b/jraft-core/target/classes/raft.desc differ
diff --git a/jraft-core/target/classes/raft.proto b/jraft-core/target/classes/raft.proto
new file mode 100644
index 0000000..040c2de
--- /dev/null
+++ b/jraft-core/target/classes/raft.proto
@@ -0,0 +1,32 @@
+syntax="proto2";
+
+package jraft;
+
+import "enum.proto";
+
+option java_package="com.alipay.sofa.jraft.entity";
+option java_outer_classname = "RaftOutter";
+
+
+message EntryMeta {
+    required int64 term = 1;
+    required EntryType type = 2;
+    repeated string peers = 3;
+    optional int64 data_len = 4;
+    // Don't change field id of `old_peers' in the consideration of backward
+    // compatibility
+    repeated string old_peers = 5;
+    // Checksum fot this log entry, since 1.2.6, added by boyan@antfin.com
+    optional int64 checksum = 6;
+    repeated string learners = 7;
+    repeated string old_learners = 8;
+};
+
+message SnapshotMeta {
+    required int64 last_included_index = 1;
+    required int64 last_included_term = 2;
+    repeated string peers = 3;
+    repeated string old_peers = 4;
+    repeated string learners = 5;
+    repeated string old_learners = 6;
+}
diff --git a/jraft-core/target/classes/rpc.proto b/jraft-core/target/classes/rpc.proto
new file mode 100644
index 0000000..87544ac
--- /dev/null
+++ b/jraft-core/target/classes/rpc.proto
@@ -0,0 +1,114 @@
+syntax="proto2";
+
+package jraft;
+import "raft.proto";
+
+option java_package="com.alipay.sofa.jraft.rpc";
+option java_outer_classname = "RpcRequests";
+
+message PingRequest {
+  required int64 send_timestamp = 1;
+}
+
+message ErrorResponse {
+  required int32 errorCode = 1;
+  optional string errorMsg = 2;
+}
+
+message InstallSnapshotRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required SnapshotMeta meta = 5;
+  required string uri = 6;
+};
+
+message InstallSnapshotResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message TimeoutNowRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+}
+
+message TimeoutNowResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+}
+
+message RequestVoteRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required int64 last_log_term = 5;
+  required int64 last_log_index = 6;
+  required bool pre_vote = 7;
+};
+
+message RequestVoteResponse {
+  required int64 term = 1;
+  required bool granted = 2;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message AppendEntriesRequestHeader {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+};
+
+message AppendEntriesRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  required string peer_id = 3;
+  required int64 term = 4;
+  required int64 prev_log_term = 5;
+  required int64 prev_log_index = 6;
+  repeated EntryMeta entries = 7;
+  required int64 committed_index = 8;
+  optional bytes data = 9;
+};
+
+message AppendEntriesResponse {
+  required int64 term = 1;
+  required bool success = 2;
+  optional int64 last_log_index = 3;
+  optional ErrorResponse errorResponse = 99;
+};
+
+message GetFileRequest {
+  required int64 reader_id = 1;
+  required string filename = 2;
+  required int64 count = 3;
+  required int64 offset = 4;
+  optional bool read_partly = 5;
+}
+
+message GetFileResponse {
+  // Data is in attachment
+  required bool eof = 1;
+  required bytes data = 2;
+  optional int64 read_size = 3;
+  optional ErrorResponse errorResponse = 99;
+}
+
+message ReadIndexRequest {
+  required string group_id = 1;
+  required string server_id = 2;
+  repeated bytes entries = 3;
+  optional string peer_id = 4;
+}
+
+message ReadIndexResponse {
+  required int64 index = 1;
+  required bool success = 2;
+  optional ErrorResponse errorResponse = 99;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_adaptiveAndPooled_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_adaptiveAndPooled_jmhTest.java
new file mode 100644
index 0000000..2337a67
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_adaptiveAndPooled_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.rpc.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_jmhType;
+public final class AppendEntriesBenchmark_adaptiveAndPooled_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult adaptiveAndPooled_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            adaptiveAndPooled_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "adaptiveAndPooled", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void adaptiveAndPooled_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.adaptiveAndPooled();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult adaptiveAndPooled_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            adaptiveAndPooled_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "adaptiveAndPooled", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void adaptiveAndPooled_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.adaptiveAndPooled();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult adaptiveAndPooled_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            adaptiveAndPooled_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.adaptiveAndPooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "adaptiveAndPooled", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void adaptiveAndPooled_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_appendentriesbenchmark0_G.adaptiveAndPooled();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult adaptiveAndPooled_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            adaptiveAndPooled_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_appendentriesbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "adaptiveAndPooled", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void adaptiveAndPooled_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_appendentriesbenchmark0_G.adaptiveAndPooled();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AppendEntriesBenchmark_jmhType f_appendentriesbenchmark0_G;
+    
+    AppendEntriesBenchmark_jmhType _jmh_tryInit_f_appendentriesbenchmark0_G(InfraControl control) throws Throwable {
+        AppendEntriesBenchmark_jmhType val = f_appendentriesbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_appendentriesbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AppendEntriesBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_appendentriesbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_copy_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_copy_jmhTest.java
new file mode 100644
index 0000000..b7c6a6d
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_copy_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.rpc.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_jmhType;
+public final class AppendEntriesBenchmark_copy_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult copy_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.copy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            copy_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.copy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "copy", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void copy_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.copy();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult copy_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.copy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            copy_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.copy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "copy", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void copy_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.copy();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult copy_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.copy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            copy_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.copy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "copy", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void copy_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_appendentriesbenchmark0_G.copy();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult copy_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            copy_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_appendentriesbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "copy", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void copy_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_appendentriesbenchmark0_G.copy();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AppendEntriesBenchmark_jmhType f_appendentriesbenchmark0_G;
+    
+    AppendEntriesBenchmark_jmhType _jmh_tryInit_f_appendentriesbenchmark0_G(InfraControl control) throws Throwable {
+        AppendEntriesBenchmark_jmhType val = f_appendentriesbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_appendentriesbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AppendEntriesBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_appendentriesbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType.java
new file mode 100644
index 0000000..4599ad9
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rpc.generated;
+public class AppendEntriesBenchmark_jmhType extends AppendEntriesBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B1.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..786e117
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rpc.generated;
+import com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark;
+public class AppendEntriesBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B2.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..96224a4
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rpc.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class AppendEntriesBenchmark_jmhType_B2 extends AppendEntriesBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AppendEntriesBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B3.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..a1583e5
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rpc.generated;
+public class AppendEntriesBenchmark_jmhType_B3 extends AppendEntriesBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_pooled_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_pooled_jmhTest.java
new file mode 100644
index 0000000..204accd
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_pooled_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.rpc.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_jmhType;
+public final class AppendEntriesBenchmark_pooled_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult pooled_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.pooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            pooled_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.pooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "pooled", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void pooled_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.pooled();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult pooled_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.pooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            pooled_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.pooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "pooled", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void pooled_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.pooled();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult pooled_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.pooled();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            pooled_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.pooled();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "pooled", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void pooled_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_appendentriesbenchmark0_G.pooled();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult pooled_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            pooled_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_appendentriesbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "pooled", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void pooled_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_appendentriesbenchmark0_G.pooled();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AppendEntriesBenchmark_jmhType f_appendentriesbenchmark0_G;
+    
+    AppendEntriesBenchmark_jmhType _jmh_tryInit_f_appendentriesbenchmark0_G(InfraControl control) throws Throwable {
+        AppendEntriesBenchmark_jmhType val = f_appendentriesbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_appendentriesbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AppendEntriesBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_appendentriesbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_zeroCopy_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_zeroCopy_jmhTest.java
new file mode 100644
index 0000000..7d6e711
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rpc/generated/AppendEntriesBenchmark_zeroCopy_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.rpc.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_jmhType;
+public final class AppendEntriesBenchmark_zeroCopy_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult zeroCopy_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.zeroCopy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            zeroCopy_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.zeroCopy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "zeroCopy", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void zeroCopy_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.zeroCopy();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult zeroCopy_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.zeroCopy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            zeroCopy_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.zeroCopy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "zeroCopy", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void zeroCopy_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_appendentriesbenchmark0_G.zeroCopy();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult zeroCopy_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_appendentriesbenchmark0_G.zeroCopy();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            zeroCopy_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_appendentriesbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_appendentriesbenchmark0_G.zeroCopy();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "zeroCopy", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void zeroCopy_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_appendentriesbenchmark0_G.zeroCopy();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult zeroCopy_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G = _jmh_tryInit_f_appendentriesbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            zeroCopy_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_appendentriesbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_appendentriesbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_appendentriesbenchmark0_G.readyTrial) {
+                            l_appendentriesbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.set(l_appendentriesbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_appendentriesbenchmark0_G_backoff = 1;
+                    while (AppendEntriesBenchmark_jmhType.tearTrialMutexUpdater.get(l_appendentriesbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_appendentriesbenchmark0_G_backoff);
+                        l_appendentriesbenchmark0_G_backoff = Math.max(1024, l_appendentriesbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_appendentriesbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "zeroCopy", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void zeroCopy_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AppendEntriesBenchmark_jmhType l_appendentriesbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_appendentriesbenchmark0_G.zeroCopy();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AppendEntriesBenchmark_jmhType f_appendentriesbenchmark0_G;
+    
+    AppendEntriesBenchmark_jmhType _jmh_tryInit_f_appendentriesbenchmark0_G(InfraControl control) throws Throwable {
+        AppendEntriesBenchmark_jmhType val = f_appendentriesbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_appendentriesbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AppendEntriesBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_appendentriesbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_defaultSingleThreadPollExecutor_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_defaultSingleThreadPollExecutor_jmhTest.java
new file mode 100644
index 0000000..2021300
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_defaultSingleThreadPollExecutor_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_defaultSingleThreadPollExecutor_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult defaultSingleThreadPollExecutor_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultSingleThreadPollExecutor_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "defaultSingleThreadPollExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultSingleThreadPollExecutor_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultSingleThreadPollExecutor_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultSingleThreadPollExecutor_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "defaultSingleThreadPollExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultSingleThreadPollExecutor_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultSingleThreadPollExecutor_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            defaultSingleThreadPollExecutor_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "defaultSingleThreadPollExecutor", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultSingleThreadPollExecutor_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultSingleThreadPollExecutor_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            defaultSingleThreadPollExecutor_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "defaultSingleThreadPollExecutor", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultSingleThreadPollExecutor_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.defaultSingleThreadPollExecutor();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType.java
new file mode 100644
index 0000000..6d77e70
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+public class SingleThreadExecutorBenchmark_jmhType extends SingleThreadExecutorBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B1.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..e603d0e
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+import com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark;
+public class SingleThreadExecutorBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B2.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..e460734
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class SingleThreadExecutorBenchmark_jmhType_B2 extends SingleThreadExecutorBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(SingleThreadExecutorBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B3.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..1bad95e
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+public class SingleThreadExecutorBenchmark_jmhType_B3 extends SingleThreadExecutorBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithConcurrentLinkedQueue_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithConcurrentLinkedQueue_jmhTest.java
new file mode 100644
index 0000000..56ae6cb
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithConcurrentLinkedQueue_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithConcurrentLinkedQueue_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithConcurrentLinkedQueue_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithConcurrentLinkedQueue_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithConcurrentLinkedQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithConcurrentLinkedQueue_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithConcurrentLinkedQueue_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithConcurrentLinkedQueue_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithConcurrentLinkedQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithConcurrentLinkedQueue_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithConcurrentLinkedQueue_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            mpscSingleThreadExecutorWithConcurrentLinkedQueue_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithConcurrentLinkedQueue", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithConcurrentLinkedQueue_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithConcurrentLinkedQueue_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            mpscSingleThreadExecutorWithConcurrentLinkedQueue_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithConcurrentLinkedQueue", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithConcurrentLinkedQueue_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithConcurrentLinkedQueue();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedBlockingQueue_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedBlockingQueue_jmhTest.java
new file mode 100644
index 0000000..2622bf5
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedBlockingQueue_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedBlockingQueue_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedBlockingQueue_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithLinkedBlockingQueue_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedBlockingQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedBlockingQueue_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedBlockingQueue_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithLinkedBlockingQueue_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedBlockingQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedBlockingQueue_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedBlockingQueue_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            mpscSingleThreadExecutorWithLinkedBlockingQueue_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedBlockingQueue", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedBlockingQueue_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedBlockingQueue_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            mpscSingleThreadExecutorWithLinkedBlockingQueue_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedBlockingQueue", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedBlockingQueue_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedBlockingQueue();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedTransferQueue_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedTransferQueue_jmhTest.java
new file mode 100644
index 0000000..17d147b
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedTransferQueue_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedTransferQueue_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedTransferQueue_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithLinkedTransferQueue_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedTransferQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedTransferQueue_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedTransferQueue_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutorWithLinkedTransferQueue_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedTransferQueue", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedTransferQueue_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedTransferQueue_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            mpscSingleThreadExecutorWithLinkedTransferQueue_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedTransferQueue", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedTransferQueue_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutorWithLinkedTransferQueue_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            mpscSingleThreadExecutorWithLinkedTransferQueue_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "mpscSingleThreadExecutorWithLinkedTransferQueue", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutorWithLinkedTransferQueue_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutorWithLinkedTransferQueue();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutor_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutor_jmhTest.java
new file mode 100644
index 0000000..54f592c
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_mpscSingleThreadExecutor_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_mpscSingleThreadExecutor_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult mpscSingleThreadExecutor_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutor_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "mpscSingleThreadExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutor_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutor_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            mpscSingleThreadExecutor_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutor_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutor_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            mpscSingleThreadExecutor_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "mpscSingleThreadExecutor", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutor_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult mpscSingleThreadExecutor_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            mpscSingleThreadExecutor_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "mpscSingleThreadExecutor", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void mpscSingleThreadExecutor_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.mpscSingleThreadExecutor();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_nettyDefaultEventExecutor_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_nettyDefaultEventExecutor_jmhTest.java
new file mode 100644
index 0000000..a91e0e7
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/concurrent/generated/SingleThreadExecutorBenchmark_nettyDefaultEventExecutor_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.util.concurrent.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_jmhType;
+public final class SingleThreadExecutorBenchmark_nettyDefaultEventExecutor_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult nettyDefaultEventExecutor_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            nettyDefaultEventExecutor_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "nettyDefaultEventExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void nettyDefaultEventExecutor_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult nettyDefaultEventExecutor_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            nettyDefaultEventExecutor_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "nettyDefaultEventExecutor", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void nettyDefaultEventExecutor_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult nettyDefaultEventExecutor_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            nettyDefaultEventExecutor_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_singlethreadexecutorbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "nettyDefaultEventExecutor", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void nettyDefaultEventExecutor_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult nettyDefaultEventExecutor_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G = _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            nettyDefaultEventExecutor_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_singlethreadexecutorbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_singlethreadexecutorbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_singlethreadexecutorbenchmark0_G.readyTrial) {
+                            l_singlethreadexecutorbenchmark0_G.tearDown();
+                            l_singlethreadexecutorbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.set(l_singlethreadexecutorbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_singlethreadexecutorbenchmark0_G_backoff = 1;
+                    while (SingleThreadExecutorBenchmark_jmhType.tearTrialMutexUpdater.get(l_singlethreadexecutorbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_singlethreadexecutorbenchmark0_G_backoff);
+                        l_singlethreadexecutorbenchmark0_G_backoff = Math.max(1024, l_singlethreadexecutorbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_singlethreadexecutorbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "nettyDefaultEventExecutor", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void nettyDefaultEventExecutor_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, SingleThreadExecutorBenchmark_jmhType l_singlethreadexecutorbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_singlethreadexecutorbenchmark0_G.nettyDefaultEventExecutor();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile SingleThreadExecutorBenchmark_jmhType f_singlethreadexecutorbenchmark0_G;
+    
+    SingleThreadExecutorBenchmark_jmhType _jmh_tryInit_f_singlethreadexecutorbenchmark0_G(InfraControl control) throws Throwable {
+        SingleThreadExecutorBenchmark_jmhType val = f_singlethreadexecutorbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_singlethreadexecutorbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new SingleThreadExecutorBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_singlethreadexecutorbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathDecode_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathDecode_jmhTest.java
new file mode 100644
index 0000000..9ecd8ad
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathDecode_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_jmhType;
+public final class AsciiCodecBenchmark_fastpathDecode_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult fastpathDecode_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            fastpathDecode_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "fastpathDecode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathDecode_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.fastpathDecode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathDecode_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            fastpathDecode_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "fastpathDecode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathDecode_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.fastpathDecode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathDecode_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            fastpathDecode_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "fastpathDecode", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathDecode_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_asciicodecbenchmark0_G.fastpathDecode();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathDecode_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            fastpathDecode_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_asciicodecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "fastpathDecode", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathDecode_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_asciicodecbenchmark0_G.fastpathDecode();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AsciiCodecBenchmark_jmhType f_asciicodecbenchmark0_G;
+    
+    AsciiCodecBenchmark_jmhType _jmh_tryInit_f_asciicodecbenchmark0_G(InfraControl control) throws Throwable {
+        AsciiCodecBenchmark_jmhType val = f_asciicodecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_asciicodecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AsciiCodecBenchmark_jmhType();
+            val.readyTrial = true;
+            f_asciicodecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathEncode_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathEncode_jmhTest.java
new file mode 100644
index 0000000..f6940ce
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_fastpathEncode_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_jmhType;
+public final class AsciiCodecBenchmark_fastpathEncode_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult fastpathEncode_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            fastpathEncode_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "fastpathEncode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathEncode_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.fastpathEncode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathEncode_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            fastpathEncode_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "fastpathEncode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathEncode_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.fastpathEncode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathEncode_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.fastpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            fastpathEncode_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.fastpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "fastpathEncode", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathEncode_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_asciicodecbenchmark0_G.fastpathEncode();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult fastpathEncode_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            fastpathEncode_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_asciicodecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "fastpathEncode", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void fastpathEncode_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_asciicodecbenchmark0_G.fastpathEncode();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AsciiCodecBenchmark_jmhType f_asciicodecbenchmark0_G;
+    
+    AsciiCodecBenchmark_jmhType _jmh_tryInit_f_asciicodecbenchmark0_G(InfraControl control) throws Throwable {
+        AsciiCodecBenchmark_jmhType val = f_asciicodecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_asciicodecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AsciiCodecBenchmark_jmhType();
+            val.readyTrial = true;
+            f_asciicodecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType.java
new file mode 100644
index 0000000..94557bf
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.util.generated;
+public class AsciiCodecBenchmark_jmhType extends AsciiCodecBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B1.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..8e3e709
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.generated;
+import com.alipay.sofa.jraft.util.AsciiCodecBenchmark;
+public class AsciiCodecBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.util.AsciiCodecBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B2.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..89d7923
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.util.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class AsciiCodecBenchmark_jmhType_B2 extends AsciiCodecBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(AsciiCodecBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B3.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..5fe1f1f
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.generated;
+public class AsciiCodecBenchmark_jmhType_B3 extends AsciiCodecBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathDecode_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathDecode_jmhTest.java
new file mode 100644
index 0000000..f25cbfe
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathDecode_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_jmhType;
+public final class AsciiCodecBenchmark_normalpathDecode_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult normalpathDecode_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            normalpathDecode_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "normalpathDecode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathDecode_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.normalpathDecode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathDecode_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            normalpathDecode_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "normalpathDecode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathDecode_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.normalpathDecode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathDecode_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathDecode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            normalpathDecode_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathDecode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "normalpathDecode", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathDecode_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_asciicodecbenchmark0_G.normalpathDecode();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathDecode_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            normalpathDecode_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_asciicodecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "normalpathDecode", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathDecode_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_asciicodecbenchmark0_G.normalpathDecode();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AsciiCodecBenchmark_jmhType f_asciicodecbenchmark0_G;
+    
+    AsciiCodecBenchmark_jmhType _jmh_tryInit_f_asciicodecbenchmark0_G(InfraControl control) throws Throwable {
+        AsciiCodecBenchmark_jmhType val = f_asciicodecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_asciicodecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AsciiCodecBenchmark_jmhType();
+            val.readyTrial = true;
+            f_asciicodecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathEncode_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathEncode_jmhTest.java
new file mode 100644
index 0000000..c8b03e0
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/AsciiCodecBenchmark_normalpathEncode_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_jmhType;
+public final class AsciiCodecBenchmark_normalpathEncode_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult normalpathEncode_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            normalpathEncode_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "normalpathEncode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathEncode_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.normalpathEncode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathEncode_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            normalpathEncode_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "normalpathEncode", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathEncode_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_asciicodecbenchmark0_G.normalpathEncode();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathEncode_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_asciicodecbenchmark0_G.normalpathEncode();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            normalpathEncode_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_asciicodecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_asciicodecbenchmark0_G.normalpathEncode();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "normalpathEncode", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathEncode_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_asciicodecbenchmark0_G.normalpathEncode();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult normalpathEncode_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G = _jmh_tryInit_f_asciicodecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            normalpathEncode_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_asciicodecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_asciicodecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "normalpathEncode", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void normalpathEncode_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, AsciiCodecBenchmark_jmhType l_asciicodecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_asciicodecbenchmark0_G.normalpathEncode();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile AsciiCodecBenchmark_jmhType f_asciicodecbenchmark0_G;
+    
+    AsciiCodecBenchmark_jmhType _jmh_tryInit_f_asciicodecbenchmark0_G(InfraControl control) throws Throwable {
+        AsciiCodecBenchmark_jmhType val = f_asciicodecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_asciicodecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new AsciiCodecBenchmark_jmhType();
+            val.readyTrial = true;
+            f_asciicodecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8Bytes_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8Bytes_jmhTest.java
new file mode 100644
index 0000000..1f5367d
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8Bytes_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_jmhType;
+public final class Utf8CodecBenchmark_defaultToUtf8Bytes_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult defaultToUtf8Bytes_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultToUtf8Bytes_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "defaultToUtf8Bytes", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8Bytes_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8Bytes_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultToUtf8Bytes_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "defaultToUtf8Bytes", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8Bytes_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8Bytes_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            defaultToUtf8Bytes_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "defaultToUtf8Bytes", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8Bytes_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8Bytes_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            defaultToUtf8Bytes_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_utf8codecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "defaultToUtf8Bytes", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8Bytes_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_utf8codecbenchmark0_G.defaultToUtf8Bytes();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile Utf8CodecBenchmark_jmhType f_utf8codecbenchmark0_G;
+    
+    Utf8CodecBenchmark_jmhType _jmh_tryInit_f_utf8codecbenchmark0_G(InfraControl control) throws Throwable {
+        Utf8CodecBenchmark_jmhType val = f_utf8codecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_utf8codecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new Utf8CodecBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_utf8codecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8String_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8String_jmhTest.java
new file mode 100644
index 0000000..f63ec2b
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_defaultToUtf8String_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_jmhType;
+public final class Utf8CodecBenchmark_defaultToUtf8String_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult defaultToUtf8String_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultToUtf8String_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "defaultToUtf8String", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8String_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.defaultToUtf8String();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8String_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            defaultToUtf8String_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "defaultToUtf8String", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8String_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.defaultToUtf8String();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8String_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.defaultToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            defaultToUtf8String_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.defaultToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "defaultToUtf8String", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8String_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_utf8codecbenchmark0_G.defaultToUtf8String();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult defaultToUtf8String_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            defaultToUtf8String_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_utf8codecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "defaultToUtf8String", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void defaultToUtf8String_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_utf8codecbenchmark0_G.defaultToUtf8String();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile Utf8CodecBenchmark_jmhType f_utf8codecbenchmark0_G;
+    
+    Utf8CodecBenchmark_jmhType _jmh_tryInit_f_utf8codecbenchmark0_G(InfraControl control) throws Throwable {
+        Utf8CodecBenchmark_jmhType val = f_utf8codecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_utf8codecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new Utf8CodecBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_utf8codecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType.java
new file mode 100644
index 0000000..78eabb0
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.util.generated;
+public class Utf8CodecBenchmark_jmhType extends Utf8CodecBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B1.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..f18932f
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.generated;
+import com.alipay.sofa.jraft.util.Utf8CodecBenchmark;
+public class Utf8CodecBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.util.Utf8CodecBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B2.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..e49a828
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.util.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class Utf8CodecBenchmark_jmhType_B2 extends Utf8CodecBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(Utf8CodecBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B3.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..d2ec355
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.util.generated;
+public class Utf8CodecBenchmark_jmhType_B3 extends Utf8CodecBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8Bytes_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8Bytes_jmhTest.java
new file mode 100644
index 0000000..e4dad16
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8Bytes_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_jmhType;
+public final class Utf8CodecBenchmark_unsafeToUtf8Bytes_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult unsafeToUtf8Bytes_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            unsafeToUtf8Bytes_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "unsafeToUtf8Bytes", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8Bytes_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8Bytes_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            unsafeToUtf8Bytes_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "unsafeToUtf8Bytes", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8Bytes_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8Bytes_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            unsafeToUtf8Bytes_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "unsafeToUtf8Bytes", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8Bytes_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8Bytes_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            unsafeToUtf8Bytes_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_utf8codecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "unsafeToUtf8Bytes", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8Bytes_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_utf8codecbenchmark0_G.unsafeToUtf8Bytes();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile Utf8CodecBenchmark_jmhType f_utf8codecbenchmark0_G;
+    
+    Utf8CodecBenchmark_jmhType _jmh_tryInit_f_utf8codecbenchmark0_G(InfraControl control) throws Throwable {
+        Utf8CodecBenchmark_jmhType val = f_utf8codecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_utf8codecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new Utf8CodecBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_utf8codecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8String_jmhTest.java b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8String_jmhTest.java
new file mode 100644
index 0000000..c5713e8
--- /dev/null
+++ b/jraft-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/util/generated/Utf8CodecBenchmark_unsafeToUtf8String_jmhTest.java
@@ -0,0 +1,447 @@
+package com.alipay.sofa.jraft.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_jmhType;
+public final class Utf8CodecBenchmark_unsafeToUtf8String_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult unsafeToUtf8String_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            unsafeToUtf8String_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "unsafeToUtf8String", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8String_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.unsafeToUtf8String();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8String_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            unsafeToUtf8String_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "unsafeToUtf8String", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8String_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_utf8codecbenchmark0_G.unsafeToUtf8String();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8String_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            unsafeToUtf8String_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_utf8codecbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_utf8codecbenchmark0_G.unsafeToUtf8String();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "unsafeToUtf8String", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8String_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_utf8codecbenchmark0_G.unsafeToUtf8String();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult unsafeToUtf8String_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G = _jmh_tryInit_f_utf8codecbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            unsafeToUtf8String_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_utf8codecbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_utf8codecbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_utf8codecbenchmark0_G.readyTrial) {
+                            l_utf8codecbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.set(l_utf8codecbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_utf8codecbenchmark0_G_backoff = 1;
+                    while (Utf8CodecBenchmark_jmhType.tearTrialMutexUpdater.get(l_utf8codecbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_utf8codecbenchmark0_G_backoff);
+                        l_utf8codecbenchmark0_G_backoff = Math.max(1024, l_utf8codecbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_utf8codecbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "unsafeToUtf8String", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void unsafeToUtf8String_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, Utf8CodecBenchmark_jmhType l_utf8codecbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_utf8codecbenchmark0_G.unsafeToUtf8String();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile Utf8CodecBenchmark_jmhType f_utf8codecbenchmark0_G;
+    
+    Utf8CodecBenchmark_jmhType _jmh_tryInit_f_utf8codecbenchmark0_G(InfraControl control) throws Throwable {
+        Utf8CodecBenchmark_jmhType val = f_utf8codecbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_utf8codecbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new Utf8CodecBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_utf8codecbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-core/target/test-classes/META-INF/BenchmarkList b/jraft-core/target/test-classes/META-INF/BenchmarkList
new file mode 100644
index 0000000..8a3c531
--- /dev/null
+++ b/jraft-core/target/test-classes/META-INF/BenchmarkList
@@ -0,0 +1,18 @@
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 110 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_mpscSingleThreadExecutor_jmhTest S 24 mpscSingleThreadExecutor S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
+JMH S 46 com.alipay.sofa.jraft.util.AsciiCodecBenchmark S 81 com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_normalpathEncode_jmhTest S 16 normalpathEncode S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 111 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_nettyDefaultEventExecutor_jmhTest S 25 nettyDefaultEventExecutor S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark S 84 com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_adaptiveAndPooled_jmhTest S 17 adaptiveAndPooled S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark S 75 com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_zeroCopy_jmhTest S 8 zeroCopy S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 46 com.alipay.sofa.jraft.util.AsciiCodecBenchmark S 79 com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_fastpathEncode_jmhTest S 14 fastpathEncode S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 45 com.alipay.sofa.jraft.util.Utf8CodecBenchmark S 83 com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_defaultToUtf8String_jmhTest S 19 defaultToUtf8String S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 46 com.alipay.sofa.jraft.util.AsciiCodecBenchmark S 79 com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_fastpathDecode_jmhTest S 14 fastpathDecode S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 133 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedBlockingQueue_jmhTest S 47 mpscSingleThreadExecutorWithLinkedBlockingQueue S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark S 71 com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_copy_jmhTest S 4 copy S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rpc.AppendEntriesBenchmark S 73 com.alipay.sofa.jraft.rpc.generated.AppendEntriesBenchmark_pooled_jmhTest S 6 pooled S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 117 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_defaultSingleThreadPollExecutor_jmhTest S 31 defaultSingleThreadPollExecutor S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
+JMH S 45 com.alipay.sofa.jraft.util.Utf8CodecBenchmark S 81 com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_unsafeToUtf8Bytes_jmhTest S 17 unsafeToUtf8Bytes S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 135 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithConcurrentLinkedQueue_jmhTest S 49 mpscSingleThreadExecutorWithConcurrentLinkedQueue S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
+JMH S 45 com.alipay.sofa.jraft.util.Utf8CodecBenchmark S 82 com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_unsafeToUtf8String_jmhTest S 18 unsafeToUtf8String S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 45 com.alipay.sofa.jraft.util.Utf8CodecBenchmark S 82 com.alipay.sofa.jraft.util.generated.Utf8CodecBenchmark_defaultToUtf8Bytes_jmhTest S 18 defaultToUtf8Bytes S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 46 com.alipay.sofa.jraft.util.AsciiCodecBenchmark S 81 com.alipay.sofa.jraft.util.generated.AsciiCodecBenchmark_normalpathDecode_jmhTest S 16 normalpathDecode S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 67 com.alipay.sofa.jraft.util.concurrent.SingleThreadExecutorBenchmark S 133 com.alipay.sofa.jraft.util.concurrent.generated.SingleThreadExecutorBenchmark_mpscSingleThreadExecutorWithLinkedTransferQueue_jmhTest S 47 mpscSingleThreadExecutorWithLinkedTransferQueue S 10 Throughput E A 1 1 1 E E E E E E E E E E E E E E U 7 SECONDS E E 
diff --git a/jraft-core/target/test-classes/META-INF/CompilerHints b/jraft-core/target/test-classes/META-INF/CompilerHints
new file mode 100644
index 0000000..97a5724
--- /dev/null
+++ b/jraft-core/target/test-classes/META-INF/CompilerHints
@@ -0,0 +1,27 @@
+dontinline,*.*_all_jmhStub
+dontinline,*.*_avgt_jmhStub
+dontinline,*.*_sample_jmhStub
+dontinline,*.*_ss_jmhStub
+dontinline,*.*_thrpt_jmhStub
+inline,com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.adaptiveAndPooled
+inline,com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.copy
+inline,com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.pooled
+inline,com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.setup
+inline,com/alipay/sofa/jraft/rpc/AppendEntriesBenchmark.zeroCopy
+inline,com/alipay/sofa/jraft/util/AsciiCodecBenchmark.fastpathDecode
+inline,com/alipay/sofa/jraft/util/AsciiCodecBenchmark.fastpathEncode
+inline,com/alipay/sofa/jraft/util/AsciiCodecBenchmark.normalpathDecode
+inline,com/alipay/sofa/jraft/util/AsciiCodecBenchmark.normalpathEncode
+inline,com/alipay/sofa/jraft/util/Utf8CodecBenchmark.defaultToUtf8Bytes
+inline,com/alipay/sofa/jraft/util/Utf8CodecBenchmark.defaultToUtf8String
+inline,com/alipay/sofa/jraft/util/Utf8CodecBenchmark.setup
+inline,com/alipay/sofa/jraft/util/Utf8CodecBenchmark.unsafeToUtf8Bytes
+inline,com/alipay/sofa/jraft/util/Utf8CodecBenchmark.unsafeToUtf8String
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.defaultSingleThreadPollExecutor
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.mpscSingleThreadExecutor
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithConcurrentLinkedQueue
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedBlockingQueue
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.mpscSingleThreadExecutorWithLinkedTransferQueue
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.nettyDefaultEventExecutor
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.setup
+inline,com/alipay/sofa/jraft/util/concurrent/SingleThreadExecutorBenchmark.tearDown
diff --git a/jraft-core/target/test-classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest b/jraft-core/target/test-classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest
new file mode 100644
index 0000000..413c621
--- /dev/null
+++ b/jraft-core/target/test-classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortTest
@@ -0,0 +1,3 @@
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl1Test
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl2Test
+com.alipay.sofa.jraft.util.JRaftServiceLoaderTest$SortImpl9Test
\ No newline at end of file
diff --git a/jraft-core/target/test-classes/log4j2.xml b/jraft-core/target/test-classes/log4j2.xml
new file mode 100644
index 0000000..d53e765
--- /dev/null
+++ b/jraft-core/target/test-classes/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-example/assembly.xml b/jraft-example/assembly.xml
new file mode 100644
index 0000000..0a96e6a
--- /dev/null
+++ b/jraft-example/assembly.xml
@@ -0,0 +1,44 @@
+
+
+    bin
+    
+        dir
+        tar.gz
+    
+    false
+    
+        
+            true
+            lib
+            runtime
+        
+    
+    
+        
+            unix
+            0755
+            
+                bin/**
+                config/**
+            
+            
+                **/src/**
+                **/target/**
+                **/.*/**
+            
+        
+    
+    
+        
+            
+                lib/
+                false
+                
+                    
+                        lib
+                    
+                
+            
+        
+    
+
diff --git a/jraft-example/bin/client_benchmark_start.sh b/jraft-example/bin/client_benchmark_start.sh
new file mode 100644
index 0000000..588dda9
--- /dev/null
+++ b/jraft-example/bin/client_benchmark_start.sh
@@ -0,0 +1,62 @@
+#! /bin/bash
+
+BASE_DIR=$(dirname $0)/..
+CLASSPATH=$(echo $BASE_DIR/lib/*.jar | tr ' ' ':')
+
+# get java version
+JAVA="$JAVA_HOME/bin/java"
+
+JAVA_VERSION=$($JAVA -version 2>&1 | awk -F\" '/version/{print $2}')
+echo "java version:$JAVA_VERSION path:$JAVA"
+
+MEMORY=$(cat /proc/meminfo |grep 'MemTotal' |awk -F : '{print $2}' |awk '{print $1}' |sed 's/^[ \t]*//g')
+echo -e "Total Memory:\n${MEMORY} KB\n"
+
+if [[ $JAVA_VERSION =~ "1.8" ]]; then
+  echo "Use java version 1.8 opt"
+
+  if (($MEMORY <= 5000000));then
+    JAVA_OPT_1="-server -Xms2000m -Xmx2000m -Xmn1000m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=1024m "
+  elif (($MEMORY <= 9000000));then
+    JAVA_OPT_1="-server -Xms3500m -Xmx3500m -Xmn1500m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=1536m "
+  elif (($MEMORY <= 17000000));then
+    JAVA_OPT_1="-server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=4g "
+  elif (($MEMORY <= 33000000));then
+    JAVA_OPT_1="-server -Xms5g -Xmx5g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=8g "
+  else
+    JAVA_OPT_1="-server -Xms6g -Xmx6g -Xmn3g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=16g "
+  fi
+
+  JAVA_OPT_2="-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSParallelRemarkEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+DisableExplicitGC"
+else
+  echo "Error, not support java version : $JAVA_VERSION"
+  exit 1
+fi
+
+JAVA_OPTS="${JAVA_OPT_1} ${JAVA_OPT_2}"
+
+JAVA_CONFIG=$(mktemp XXXXXXXX)
+cat < $JAVA_CONFIG
+${JAVA_OPTS}
+-cp $CLASSPATH
+com.alipay.sofa.jraft.benchmark.BenchmarkBootstrap
+client
+$1
+$2
+$3
+$4
+$5
+$6
+$7
+$8
+EOF
+
+JAVA_CONFIG=$(cat $JAVA_CONFIG | xargs echo)
+
+echo $JAVA_CONFIG
+
+JAVA="$JAVA_HOME/bin/java"
+
+HOSTNAME=`hostname`
+
+nohup $JAVA $JAVA_CONFIG >> client_stdout 2>&1 &
diff --git a/jraft-example/bin/server_benchmark_start.sh b/jraft-example/bin/server_benchmark_start.sh
new file mode 100644
index 0000000..18a0711
--- /dev/null
+++ b/jraft-example/bin/server_benchmark_start.sh
@@ -0,0 +1,56 @@
+#! /bin/bash
+
+BASE_DIR=$(dirname $0)/..
+CLASSPATH=$(echo $BASE_DIR/lib/*.jar | tr ' ' ':')
+
+# get java version
+JAVA="$JAVA_HOME/bin/java"
+
+JAVA_VERSION=$($JAVA -version 2>&1 | awk -F\" '/version/{print $2}')
+echo "java version:$JAVA_VERSION path:$JAVA"
+
+MEMORY=$(cat /proc/meminfo |grep 'MemTotal' |awk -F : '{print $2}' |awk '{print $1}' |sed 's/^[ \t]*//g')
+echo -e "Total Memory:\n${MEMORY} KB\n"
+
+if [[ $JAVA_VERSION =~ "1.8" ]]; then
+  echo "Use java version 1.8 opt"
+
+  if (($MEMORY <= 5000000));then
+    JAVA_OPT_1="-server -Xms2000m -Xmx2000m -Xmn1000m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=1024m "
+  elif (($MEMORY <= 9000000));then
+    JAVA_OPT_1="-server -Xms3500m -Xmx3500m -Xmn1500m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=1536m "
+  elif (($MEMORY <= 17000000));then
+    JAVA_OPT_1="-server -Xms4g -Xmx4g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=4g "
+  elif (($MEMORY <= 33000000));then
+    JAVA_OPT_1="-server -Xms5g -Xmx5g -Xmn2g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=8g "
+  else
+    JAVA_OPT_1="-server -Xms6g -Xmx6g -Xmn3g -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m -Xss256k -XX:MaxDirectMemorySize=16g "
+  fi
+
+  JAVA_OPT_2="-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSParallelRemarkEnabled -XX:+UseCMSInitiatingOccupancyOnly -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+CMSClassUnloadingEnabled -XX:+DisableExplicitGC"
+else
+  echo "Error, not support java version : $JAVA_VERSION"
+  exit 1
+fi
+
+JAVA_OPTS="${JAVA_OPT_1} ${JAVA_OPT_2}"
+
+JAVA_CONFIG=$(mktemp XXXXXXXX)
+cat < $JAVA_CONFIG
+${JAVA_OPTS}
+-cp $CLASSPATH
+com.alipay.sofa.jraft.benchmark.BenchmarkBootstrap
+server
+$1
+$2
+EOF
+
+JAVA_CONFIG=$(cat $JAVA_CONFIG | xargs echo)
+
+echo $JAVA_CONFIG
+
+JAVA="$JAVA_HOME/bin/java"
+
+HOSTNAME=`hostname`
+
+nohup $JAVA $JAVA_CONFIG >> server_stdout 2>&1 &
diff --git a/jraft-example/bin/shutdown.sh b/jraft-example/bin/shutdown.sh
new file mode 100644
index 0000000..4371c72
--- /dev/null
+++ b/jraft-example/bin/shutdown.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+echo "Prepare to kill benchmark process"
+
+pid=`ps ax | grep -i 'com.alipay.sofa.jraft.benchmark.BenchmarkBootstrap' | grep java | awk '{print $1}'`
+if [ ! -n "$pid" ] ; then
+   echo "$LOG_PREFIX no java process running"
+else
+   kill $pid
+   echo $LOG_PREFIX kill $pid
+fi
+
+if [ ! -n "$pid" ] ; then
+	echo "$LOG_PREFIX kill successfully"
+else
+    kill -9 $pid
+    sleep 1
+    echo "$LOG_PREFIX kill process $pid successfully"
+fi
\ No newline at end of file
diff --git a/jraft-example/config/benchmark_client.yaml b/jraft-example/config/benchmark_client.yaml
new file mode 100644
index 0000000..1052898
--- /dev/null
+++ b/jraft-example/config/benchmark_client.yaml
@@ -0,0 +1,42 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+  regionRouteTableOptionsList:
+    - { regionId:  1,                      endKey: 01000000 }
+    - { regionId:  2, startKey: 01000000,  endKey: 02000000 }
+    - { regionId:  3, startKey: 02000000,  endKey: 03000000 }
+    - { regionId:  4, startKey: 03000000,  endKey: 04000000 }
+    - { regionId:  5, startKey: 04000000,  endKey: 05000000 }
+    - { regionId:  6, startKey: 05000000,  endKey: 06000000 }
+    - { regionId:  7, startKey: 06000000,  endKey: 07000000 }
+    - { regionId:  8, startKey: 07000000,  endKey: 08000000 }
+    - { regionId:  9, startKey: 08000000,  endKey: 09000000 }
+    - { regionId: 10, startKey: 09000000,  endKey: 10000000 }
+    - { regionId: 11, startKey: 10000000,  endKey: 11000000 }
+    - { regionId: 12, startKey: 11000000,  endKey: 12000000 }
+    - { regionId: 13, startKey: 12000000,  endKey: 13000000 }
+    - { regionId: 14, startKey: 13000000,  endKey: 14000000 }
+    - { regionId: 15, startKey: 14000000,  endKey: 15000000 }
+    - { regionId: 16, startKey: 15000000,  endKey: 16000000 }
+    - { regionId: 17, startKey: 16000000,  endKey: 17000000 }
+    - { regionId: 18, startKey: 17000000,  endKey: 18000000 }
+    - { regionId: 19, startKey: 18000000,  endKey: 19000000 }
+    - { regionId: 20, startKey: 19000000,  endKey: 20000000 }
+    - { regionId: 21, startKey: 20000000,  endKey: 21000000 }
+    - { regionId: 22, startKey: 21000000,  endKey: 22000000 }
+    - { regionId: 23, startKey: 22000000,  endKey: 23000000 }
+    - { regionId: 24, startKey: 23000000,                   }
+
+onlyLeaderRead: true
+
+rpcOptions:
+  callbackExecutorCorePoolSize: 0
+  callbackExecutorMaximumPoolSize: 0
+  rpcTimeoutMillis: 2000
+
+failoverRetries: 0
+useParallelKVExecutor: false
+
diff --git a/jraft-example/config/benchmark_server.yaml b/jraft-example/config/benchmark_server.yaml
new file mode 100644
index 0000000..73e44fe
--- /dev/null
+++ b/jraft-example/config/benchmark_server.yaml
@@ -0,0 +1,51 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  storageType: Memory
+  rocksDBOptions:
+    fastSnapshot: true
+    dbPath: rhea_benchmark_db
+  raftDataPath: rhea_benchmark_raft_data
+  raftRpcCoreThreads: 128
+  serverAddress:
+    port: 18091
+  commonNodeOptions:
+    raftOptions:
+      readOnlyOptions: ReadOnlyLeaseBased
+  regionEngineOptionsList:
+    - { regionId:  1,                      endKey: 01000000 }
+    - { regionId:  2, startKey: 01000000,  endKey: 02000000 }
+    - { regionId:  3, startKey: 02000000,  endKey: 03000000 }
+    - { regionId:  4, startKey: 03000000,  endKey: 04000000 }
+    - { regionId:  5, startKey: 04000000,  endKey: 05000000 }
+    - { regionId:  6, startKey: 05000000,  endKey: 06000000 }
+    - { regionId:  7, startKey: 06000000,  endKey: 07000000 }
+    - { regionId:  8, startKey: 07000000,  endKey: 08000000 }
+    - { regionId:  9, startKey: 08000000,  endKey: 09000000 }
+    - { regionId: 10, startKey: 09000000,  endKey: 10000000 }
+    - { regionId: 11, startKey: 10000000,  endKey: 11000000 }
+    - { regionId: 12, startKey: 11000000,  endKey: 12000000 }
+    - { regionId: 13, startKey: 12000000,  endKey: 13000000 }
+    - { regionId: 14, startKey: 13000000,  endKey: 14000000 }
+    - { regionId: 15, startKey: 14000000,  endKey: 15000000 }
+    - { regionId: 16, startKey: 15000000,  endKey: 16000000 }
+    - { regionId: 17, startKey: 16000000,  endKey: 17000000 }
+    - { regionId: 18, startKey: 17000000,  endKey: 18000000 }
+    - { regionId: 19, startKey: 18000000,  endKey: 19000000 }
+    - { regionId: 20, startKey: 19000000,  endKey: 20000000 }
+    - { regionId: 21, startKey: 20000000,  endKey: 21000000 }
+    - { regionId: 22, startKey: 21000000,  endKey: 22000000 }
+    - { regionId: 23, startKey: 22000000,  endKey: 23000000 }
+    - { regionId: 24, startKey: 23000000,                   }
+
+onlyLeaderRead: true
+
+failoverRetries: 0
+futureTimeoutMillis: 2000
+useParallelKVExecutor: false
+
diff --git a/jraft-example/pom.xml b/jraft-example/pom.xml
new file mode 100644
index 0000000..9b78a25
--- /dev/null
+++ b/jraft-example/pom.xml
@@ -0,0 +1,110 @@
+
+
+
+    4.0.0
+    
+        jraft-parent
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+    jraft-example
+    jar
+    jraft-example ${project.version}
+
+    
+        2.10.5.1
+        2.10.4
+    
+
+    
+        
+            ${project.groupId}
+            jraft-core
+        
+        
+            ${project.groupId}
+            jraft-rheakv-core
+        
+        
+            org.apache.logging.log4j
+            log4j-api
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-core
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-slf4j-impl
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-jcl
+            ${log4j.version}
+        
+        
+            com.fasterxml.jackson.dataformat
+            jackson-dataformat-yaml
+            ${jackson.dataformat.version}
+        
+        
+            com.fasterxml.jackson.core
+            jackson-databind
+            ${jackson.databind.version}
+        
+    
+
+    
+        
+            
+                org.apache.maven.plugins
+                maven-compiler-plugin
+                3.1
+                
+                    1.8
+                    1.8
+                
+            
+            
+                org.apache.maven.plugins
+                maven-surefire-plugin
+                
+                    true
+                
+            
+            
+                org.codehaus.mojo
+                exec-maven-plugin
+                
+                    java
+                    com.alipay.sofa.jraft.benchmark.BenchmarkBootstrap
+                
+            
+            
+                org.apache.maven.plugins
+                maven-jar-plugin
+            
+            
+                org.apache.maven.plugins
+                maven-assembly-plugin
+                
+                    
+                        make-assembly
+                        package
+                        
+                            single
+                        
+                    
+                
+                
+                    
+                        assembly.xml
+                    
+                
+            
+        
+    
+
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/BenchmarkBootstrap.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/BenchmarkBootstrap.java
new file mode 100644
index 0000000..4e8a134
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/BenchmarkBootstrap.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.benchmark;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.benchmark.client.BenchmarkClient;
+import com.alipay.sofa.jraft.benchmark.server.BenchmarkServer;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BenchmarkBootstrap {
+
+    private static final Logger LOG = LoggerFactory.getLogger(BenchmarkBootstrap.class);
+
+    public static void main(final String[] args) {
+        if (args.length < 1) {
+            LOG.error("Invalid args.");
+            System.exit(-1);
+        }
+        final String who = args[0];
+        if ("client".equals(who)) {
+            BenchmarkClient.main(args);
+        } else if ("server".equals(who)) {
+            BenchmarkServer.main(args);
+        } else {
+            LOG.error("Invalid args[0]: {}", who);
+            System.exit(-1);
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/Yaml.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/Yaml.java
new file mode 100644
index 0000000..4ff2076
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/Yaml.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.benchmark;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class Yaml {
+
+    public static void main(final String[] args) {
+        readConfig(Paths.get("jraft-example", "config", "benchmark_client.yaml").toString());
+        readConfig(Paths.get("jraft-example", "config", "benchmark_server.yaml").toString());
+    }
+
+    public static RheaKVStoreOptions readConfig(final String name) {
+        final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        final RheaKVStoreOptions opts;
+        try {
+            opts = mapper.readValue(new File(name), RheaKVStoreOptions.class);
+            System.out.println(opts);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        return opts;
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/client/BenchmarkClient.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/client/BenchmarkClient.java
new file mode 100644
index 0000000..c081aa3
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/client/BenchmarkClient.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.benchmark.client;
+
+import java.util.ArrayDeque;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.benchmark.Yaml;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.options.RegionRouteTableOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.codahale.metrics.ConsoleReporter;
+import com.codahale.metrics.Timer;
+
+/**
+ * @author jiachun.fjc
+ */
+public class BenchmarkClient {
+
+    private static final Logger LOG      = LoggerFactory.getLogger(BenchmarkClient.class);
+
+    private static final byte[] BYTES    = new byte[] { 0, 1 };
+    private static final Timer  putTimer = KVMetrics.timer("put_benchmark_timer");
+    private static final Timer  getTimer = KVMetrics.timer("get_benchmark_timer");
+    private static final Timer  timer    = KVMetrics.timer("benchmark_timer");
+
+    public static void main(final String[] args) {
+        if (args.length < 7) {
+            LOG.error("Args: [initialServerList], [configPath], [threads], [writeRatio], [readRatio], [valueSize] are needed.");
+            System.exit(-1);
+        }
+        final String initialServerList = args[1];
+        final String configPath = args[2];
+        final int threads = Integer.parseInt(args[3]);
+        final int writeRatio = Integer.parseInt(args[4]);
+        final int readRatio = Integer.parseInt(args[5]);
+        final int valueSize = Integer.parseInt(args[6]);
+
+        final RheaKVStoreOptions opts = Yaml.readConfig(configPath);
+        opts.setInitialServerList(initialServerList);
+        final RheaKVStore rheaKVStore = new DefaultRheaKVStore();
+        if (!rheaKVStore.init(opts)) {
+            LOG.error("Fail to init [RheaKVStore]");
+            System.exit(-1);
+        }
+
+        final List regionRouteTableOptionsList = opts.getPlacementDriverOptions()
+            .getRegionRouteTableOptionsList();
+
+        rebalance(rheaKVStore, initialServerList, regionRouteTableOptionsList);
+
+        rheaKVStore.bPut("benchmark", BytesUtil.writeUtf8("benchmark start at: " + new Date()));
+        LOG.info(BytesUtil.readUtf8(rheaKVStore.bGet("benchmark")));
+
+        ConsoleReporter.forRegistry(KVMetrics.metricRegistry()) //
+            .build() //
+            .start(30, TimeUnit.SECONDS);
+
+        LOG.info("Start benchmark...");
+        startBenchmark(rheaKVStore, threads, writeRatio, readRatio, valueSize, regionRouteTableOptionsList);
+    }
+
+    public static void startBenchmark(final RheaKVStore rheaKVStore, final int threads, final int writeRatio, final int readRatio,
+                                      final int valueSize, final List regionRouteTableOptionsList) {
+        for (int i = 0; i < threads; i++) {
+            final Thread t = new Thread(() -> doRequest(rheaKVStore, writeRatio, readRatio, valueSize, regionRouteTableOptionsList));
+            t.setDaemon(true);
+            t.start();
+        }
+    }
+
+    @SuppressWarnings("InfiniteLoopStatement")
+    public static void doRequest(final RheaKVStore rheaKVStore, final int writeRatio, final int readRatio, final int valueSize,
+                                 final List regionRouteTableOptionsList) {
+        final int regionSize = regionRouteTableOptionsList.size();
+        final ThreadLocalRandom random = ThreadLocalRandom.current();
+        final int sum = writeRatio + readRatio;
+        final Semaphore slidingWindow = new Semaphore(sum);
+        int index = 0;
+        int randomRegionIndex = 0;
+        final byte[] valeBytes = new byte[valueSize];
+        random.nextBytes(valeBytes);
+        for (;;) {
+            try {
+                slidingWindow.acquire();
+            } catch (final Exception e) {
+                LOG.error("Wrong slidingWindow: {}, {}", slidingWindow.toString(), StackTraceUtil.stackTrace(e));
+            }
+            int i = index++;
+            if (i % sum == 0) {
+                randomRegionIndex = random.nextInt(regionSize);
+            }
+            byte[] keyBytes = regionRouteTableOptionsList.get(randomRegionIndex).getStartKeyBytes();
+            if (keyBytes == null) {
+                keyBytes = BYTES;
+            }
+            final Timer.Context ctx = timer.time();
+            if (Math.abs(i % sum) < writeRatio) {
+                // put
+                final Timer.Context putCtx = putTimer.time();
+                final CompletableFuture f = put(rheaKVStore, keyBytes, valeBytes);
+                f.whenComplete((ignored, throwable) -> {
+                    slidingWindow.release();
+                    ctx.stop();
+                    putCtx.stop();
+                });
+            } else {
+                // get
+                final Timer.Context getCtx = getTimer.time();
+                final CompletableFuture f = get(rheaKVStore, keyBytes);
+                f.whenComplete((ignored, throwable) -> {
+                    slidingWindow.release();
+                    ctx.stop();
+                    getCtx.stop();
+                });
+            }
+        }
+    }
+
+    public static CompletableFuture put(final RheaKVStore rheaKVStore, final byte[] key, final byte[] value) {
+        return rheaKVStore.put(key, value);
+    }
+
+    public static CompletableFuture get(final RheaKVStore rheaKVStore, final byte[] key) {
+        return rheaKVStore.get(key);
+    }
+
+    // Because we use fake PD, so we need manual rebalance
+    public static void rebalance(final RheaKVStore rheaKVStore, final String initialServerList,
+                                 final List regionRouteTableOptionsList) {
+        final PlacementDriverClient pdClient = rheaKVStore.getPlacementDriverClient();
+        final Configuration configuration = new Configuration();
+        configuration.parse(initialServerList);
+        final int serverSize = configuration.size();
+        final int regionSize = regionRouteTableOptionsList.size();
+        final int regionSizePerServer = regionSize / serverSize;
+        final Queue regions = new ArrayDeque<>();
+        for (final RegionRouteTableOptions r : regionRouteTableOptionsList) {
+            regions.add(r.getRegionId());
+        }
+        final Map peerMap = Maps.newHashMap();
+        for (;;) {
+            final Long regionId = regions.poll();
+            if (regionId == null) {
+                break;
+            }
+            PeerId peerId;
+            try {
+                final Endpoint endpoint = pdClient.getLeader(regionId, true, 10000);
+                if (endpoint == null) {
+                    continue;
+                }
+                peerId = new PeerId(endpoint, 0);
+                LOG.info("Region {} leader is {}", regionId, peerId);
+            } catch (final Exception e) {
+                regions.add(regionId);
+                continue;
+            }
+            final Integer size = peerMap.get(peerId);
+            if (size == null) {
+                peerMap.put(peerId, 1);
+                continue;
+            }
+            if (size < regionSizePerServer) {
+                peerMap.put(peerId, size + 1);
+                continue;
+            }
+            for (final PeerId p : configuration.listPeers()) {
+                final Integer pSize = peerMap.get(p);
+                if (pSize != null && pSize >= regionSizePerServer) {
+                    continue;
+                }
+                try {
+                    pdClient.transferLeader(regionId, JRaftHelper.toPeer(p), true);
+                    LOG.info("Region {} transfer leader to {}", regionId, p);
+                    regions.add(regionId);
+                    break;
+                } catch (final Exception e) {
+                    LOG.error("Fail to transfer leader to {}", p);
+                }
+            }
+        }
+
+        for (final RegionRouteTableOptions r : regionRouteTableOptionsList) {
+            final Long regionId = r.getRegionId();
+            try {
+                final Endpoint endpoint = pdClient.getLeader(regionId, true, 10000);
+                LOG.info("Finally, the region: {} leader is: {}", regionId, endpoint);
+            } catch (final Exception e) {
+                LOG.error("Fail to get leader: {}", StackTraceUtil.stackTrace(e));
+            }
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/server/BenchmarkServer.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/server/BenchmarkServer.java
new file mode 100644
index 0000000..f9bb007
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/benchmark/server/BenchmarkServer.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.benchmark.server;
+
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.benchmark.Yaml;
+import com.alipay.sofa.jraft.example.rheakv.Node;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.codahale.metrics.ConsoleReporter;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BenchmarkServer {
+
+    private static final Logger LOG = LoggerFactory.getLogger(BenchmarkServer.class);
+
+    public static void main(final String[] args) {
+        if (args.length < 3) {
+            LOG.error("[initialServerList], [configPath] are needed.");
+        }
+        final String initialServerList = args[1];
+        final String configPath = args[2];
+
+        final RheaKVStoreOptions opts = Yaml.readConfig(configPath);
+        opts.setInitialServerList(initialServerList);
+
+        final Node node = new Node(opts);
+        node.start();
+
+        ConsoleReporter.forRegistry(KVMetrics.metricRegistry()) //
+                .build() //
+                .start(30, TimeUnit.SECONDS);
+
+        Runtime.getRuntime().addShutdownHook(new Thread(node::stop));
+        LOG.info("BenchmarkServer start OK, options: {}", opts);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClient.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClient.java
new file mode 100644
index 0000000..2002617
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClient.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RemotingException;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.IncrementAndGetRequest;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterGrpcHelper;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executor;
+
+public class CounterClient {
+
+    public static void main(final String[] args) throws Exception {
+        if (args.length != 2) {
+            System.out.println("Usage : java com.alipay.sofa.jraft.example.counter.CounterClient {groupId} {conf}");
+            System.out
+                .println("Example: java com.alipay.sofa.jraft.example.counter.CounterClient counter 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083");
+            System.exit(1);
+        }
+        final String groupId = args[0];
+        final String confStr = args[1];
+        CounterGrpcHelper.initGRpc();
+
+        final Configuration conf = new Configuration();
+        if (!conf.parse(confStr)) {
+            throw new IllegalArgumentException("Fail to parse conf:" + confStr);
+        }
+
+        RouteTable.getInstance().updateConfiguration(groupId, conf);
+
+        final CliClientServiceImpl cliClientService = new CliClientServiceImpl();
+        cliClientService.init(new CliOptions());
+
+        if (!RouteTable.getInstance().refreshLeader(cliClientService, groupId, 1000).isOk()) {
+            throw new IllegalStateException("Refresh leader failed");
+        }
+
+        final PeerId leader = RouteTable.getInstance().selectLeader(groupId);
+        System.out.println("Leader is " + leader);
+        final int n = 1000;
+        final CountDownLatch latch = new CountDownLatch(n);
+        final long start = System.currentTimeMillis();
+        for (int i = 0; i < n; i++) {
+            incrementAndGet(cliClientService, leader, i, latch);
+        }
+        latch.await();
+        System.out.println(n + " ops, cost : " + (System.currentTimeMillis() - start) + " ms.");
+        System.exit(0);
+    }
+
+    private static void incrementAndGet(final CliClientServiceImpl cliClientService, final PeerId leader,
+                                        final long delta, CountDownLatch latch) throws RemotingException,
+                                                                               InterruptedException {
+        IncrementAndGetRequest request = IncrementAndGetRequest.newBuilder().setDelta(delta).build();
+        cliClientService.getRpcClient().invokeAsync(leader.getEndpoint(), request, new InvokeCallback() {
+
+            @Override
+            public void complete(Object result, Throwable err) {
+                if (err == null) {
+                    latch.countDown();
+                    System.out.println("incrementAndGet result:" + result);
+                } else {
+                    err.printStackTrace();
+                    latch.countDown();
+                }
+            }
+
+            @Override
+            public Executor executor() {
+                return null;
+            }
+        }, 5000);
+    }
+
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClosure.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClosure.java
new file mode 100644
index 0000000..95f0e14
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterClosure.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.ValueResponse;
+
+/**
+ * @author likun (saimu.msm@antfin.com)
+ */
+public abstract class CounterClosure implements Closure {
+
+    private ValueResponse    valueResponse;
+    private CounterOperation counterOperation;
+
+    public void setCounterOperation(CounterOperation counterOperation) {
+        this.counterOperation = counterOperation;
+    }
+
+    public CounterOperation getCounterOperation() {
+        return counterOperation;
+    }
+
+    public ValueResponse getValueResponse() {
+        return valueResponse;
+    }
+
+    public void setValueResponse(ValueResponse valueResponse) {
+        this.valueResponse = valueResponse;
+    }
+
+    protected void failure(final String errorMsg, final String redirect) {
+        final ValueResponse response = ValueResponse.newBuilder().setSuccess(false).setErrorMsg(errorMsg)
+            .setRedirect(redirect).build();
+        setValueResponse(response);
+    }
+
+    protected void success(final long value) {
+        final ValueResponse response = ValueResponse.newBuilder().setValue(value).setSuccess(true).build();
+        setValueResponse(response);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterOperation.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterOperation.java
new file mode 100644
index 0000000..1d2d397
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterOperation.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import java.io.Serializable;
+
+/**
+ * The counter operation
+ *
+ * @author likun (saimu.msm@antfin.com)
+ */
+public class CounterOperation implements Serializable {
+
+    private static final long serialVersionUID = -6597003954824547294L;
+
+    /** Get value */
+    public static final byte  GET              = 0x01;
+    /** Increment and get value */
+    public static final byte  INCREMENT        = 0x02;
+
+    private byte              op;
+    private long              delta;
+
+    public static CounterOperation createGet() {
+        return new CounterOperation(GET);
+    }
+
+    public static CounterOperation createIncrement(final long delta) {
+        return new CounterOperation(INCREMENT, delta);
+    }
+
+    public CounterOperation(byte op) {
+        this(op, 0);
+    }
+
+    public CounterOperation(byte op, long delta) {
+        this.op = op;
+        this.delta = delta;
+    }
+
+    public byte getOp() {
+        return op;
+    }
+
+    public long getDelta() {
+        return delta;
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServer.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServer.java
new file mode 100644
index 0000000..cedd926
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServer.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.ValueResponse;
+import com.alipay.sofa.jraft.example.counter.rpc.GetValueRequestProcessor;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterGrpcHelper;
+import com.alipay.sofa.jraft.example.counter.rpc.IncrementAndGetRequestProcessor;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Counter server that keeps a counter value in a raft group.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ * 

+ * 2018-Apr-09 4:51:02 PM
+ */
+public class CounterServer {
+
+    private RaftGroupService    raftGroupService;
+    private Node                node;
+    private CounterStateMachine fsm;
+
+    public CounterServer(final String dataPath, final String groupId, final PeerId serverId,
+                         final NodeOptions nodeOptions) throws IOException {
+        // init raft data path, it contains log,meta,snapshot
+        FileUtils.forceMkdir(new File(dataPath));
+
+        // here use same RPC server for raft and business. It also can be seperated generally
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint());
+        // GrpcServer need init marshaller
+        CounterGrpcHelper.initGRpc();
+        CounterGrpcHelper.setRpcServer(rpcServer);
+
+        // register business processor
+        CounterService counterService = new CounterServiceImpl(this);
+        rpcServer.registerProcessor(new GetValueRequestProcessor(counterService));
+        rpcServer.registerProcessor(new IncrementAndGetRequestProcessor(counterService));
+        // init state machine
+        this.fsm = new CounterStateMachine();
+        // set fsm to nodeOptions
+        nodeOptions.setFsm(this.fsm);
+        // set storage path (log,meta,snapshot)
+        // log, must
+        nodeOptions.setLogUri(dataPath + File.separator + "log");
+        // meta, must
+        nodeOptions.setRaftMetaUri(dataPath + File.separator + "raft_meta");
+        // snapshot, optional, generally recommended
+        nodeOptions.setSnapshotUri(dataPath + File.separator + "snapshot");
+        // init raft group service framework
+        this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOptions, rpcServer);
+        // start raft node
+        this.node = this.raftGroupService.start();
+    }
+
+    public CounterStateMachine getFsm() {
+        return this.fsm;
+    }
+
+    public Node getNode() {
+        return this.node;
+    }
+
+    public RaftGroupService RaftGroupService() {
+        return this.raftGroupService;
+    }
+
+    /**
+     * Redirect request to new leader
+     */
+    public ValueResponse redirect() {
+        final ValueResponse.Builder builder = ValueResponse.newBuilder().setSuccess(false);
+        if (this.node != null) {
+            final PeerId leader = this.node.getLeaderId();
+            if (leader != null) {
+                builder.setRedirect(leader.toString());
+            }
+        }
+        return builder.build();
+    }
+
+    public static void main(final String[] args) throws IOException {
+        if (args.length != 4) {
+            System.out
+                .println("Usage : java com.alipay.sofa.jraft.example.counter.CounterServer {dataPath} {groupId} {serverId} {initConf}");
+            System.out
+                .println("Example: java com.alipay.sofa.jraft.example.counter.CounterServer /tmp/server1 counter 127.0.0.1:8081 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083");
+            System.exit(1);
+        }
+        final String dataPath = args[0];
+        final String groupId = args[1];
+        final String serverIdStr = args[2];
+        final String initConfStr = args[3];
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        // for test, modify some params
+        // set election timeout to 1s
+        nodeOptions.setElectionTimeoutMs(1000);
+        // disable CLI service。
+        nodeOptions.setDisableCli(false);
+        // do snapshot every 30s
+        nodeOptions.setSnapshotIntervalSecs(30);
+        // parse server address
+        final PeerId serverId = new PeerId();
+        if (!serverId.parse(serverIdStr)) {
+            throw new IllegalArgumentException("Fail to parse serverId:" + serverIdStr);
+        }
+        final Configuration initConf = new Configuration();
+        if (!initConf.parse(initConfStr)) {
+            throw new IllegalArgumentException("Fail to parse initConf:" + initConfStr);
+        }
+        // set cluster configuration
+        nodeOptions.setInitialConf(initConf);
+
+        // start raft server
+        final CounterServer counterServer = new CounterServer(dataPath, groupId, serverId, nodeOptions);
+        System.out.println("Started counter server at port:"
+                           + counterServer.getNode().getNodeId().getPeerId().getPort());
+        // GrpcServer need block to prevent process exit
+        CounterGrpcHelper.blockUntilShutdown();
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterService.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterService.java
new file mode 100644
index 0000000..98ca6be
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterService.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+/**
+ * The counter service supporting query and count function.
+ *
+ * @author likun (saimu.msm@antfin.com)
+ */
+public interface CounterService {
+
+    /**
+     * Get current value from counter
+     *
+     * Provide consistent reading if {@code readOnlySafe} is true.
+     */
+    void get(final boolean readOnlySafe, final CounterClosure closure);
+
+    /**
+     * Add delta to counter then get value
+     */
+    void incrementAndGet(final long delta, final CounterClosure closure);
+}
\ No newline at end of file
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServiceImpl.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServiceImpl.java
new file mode 100644
index 0000000..2dbe9be
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterServiceImpl.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.Executor;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.remoting.exception.CodecException;
+import com.alipay.remoting.serialization.SerializerManager;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rhea.StoreEngineHelper;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ * @author likun (saimu.msm@antfin.com)
+ */
+public class CounterServiceImpl implements CounterService {
+    private static final Logger LOG = LoggerFactory.getLogger(CounterServiceImpl.class);
+
+    private final CounterServer counterServer;
+    private final Executor      readIndexExecutor;
+
+    public CounterServiceImpl(CounterServer counterServer) {
+        this.counterServer = counterServer;
+        this.readIndexExecutor = createReadIndexExecutor();
+    }
+
+    private Executor createReadIndexExecutor() {
+        final StoreEngineOptions opts = new StoreEngineOptions();
+        return StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads());
+    }
+
+    @Override
+    public void get(final boolean readOnlySafe, final CounterClosure closure) {
+        if(!readOnlySafe){
+            closure.success(getValue());
+            closure.run(Status.OK());
+            return;
+        }
+
+        this.counterServer.getNode().readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+            @Override
+            public void run(Status status, long index, byte[] reqCtx) {
+                if(status.isOk()){
+                    closure.success(getValue());
+                    closure.run(Status.OK());
+                    return;
+                }
+                CounterServiceImpl.this.readIndexExecutor.execute(() -> {
+                    if(isLeader()){
+                        LOG.debug("Fail to get value with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        applyOperation(CounterOperation.createGet(), closure);
+                    }else {
+                        handlerNotLeaderError(closure);
+                    }
+                });
+            }
+        });
+    }
+
+    private boolean isLeader() {
+        return this.counterServer.getFsm().isLeader();
+    }
+
+    private long getValue() {
+        return this.counterServer.getFsm().getValue();
+    }
+
+    private String getRedirect() {
+        return this.counterServer.redirect().getRedirect();
+    }
+
+    @Override
+    public void incrementAndGet(final long delta, final CounterClosure closure) {
+        applyOperation(CounterOperation.createIncrement(delta), closure);
+    }
+
+    private void applyOperation(final CounterOperation op, final CounterClosure closure) {
+        if (!isLeader()) {
+            handlerNotLeaderError(closure);
+            return;
+        }
+
+        try {
+            closure.setCounterOperation(op);
+            final Task task = new Task();
+            task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(op)));
+            task.setDone(closure);
+            this.counterServer.getNode().apply(task);
+        } catch (CodecException e) {
+            String errorMsg = "Fail to encode CounterOperation";
+            LOG.error(errorMsg, e);
+            closure.failure(errorMsg, StringUtils.EMPTY);
+            closure.run(new Status(RaftError.EINTERNAL, errorMsg));
+        }
+    }
+
+    private void handlerNotLeaderError(final CounterClosure closure) {
+        closure.failure("Not leader.", getRedirect());
+        closure.run(new Status(RaftError.EPERM, "Not leader"));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterStateMachine.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterStateMachine.java
new file mode 100644
index 0000000..40b07f4
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/CounterStateMachine.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter;
+
+import static com.alipay.sofa.jraft.example.counter.CounterOperation.GET;
+import static com.alipay.sofa.jraft.example.counter.CounterOperation.INCREMENT;
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alipay.remoting.exception.CodecException;
+import com.alipay.remoting.serialization.SerializerManager;
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.example.counter.snapshot.CounterSnapshotFile;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Counter state machine.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 4:52:31 PM
+ */
+public class CounterStateMachine extends StateMachineAdapter {
+
+    private static final Logger LOG        = LoggerFactory.getLogger(CounterStateMachine.class);
+
+    /**
+     * Counter value
+     */
+    private final AtomicLong    value      = new AtomicLong(0);
+    /**
+     * Leader term
+     */
+    private final AtomicLong    leaderTerm = new AtomicLong(-1);
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    /**
+     * Returns current value.
+     */
+    public long getValue() {
+        return this.value.get();
+    }
+
+    @Override
+    public void onApply(final Iterator iter) {
+        while (iter.hasNext()) {
+            long current = 0;
+            CounterOperation counterOperation = null;
+
+            CounterClosure closure = null;
+            if (iter.done() != null) {
+                // This task is applied by this node, get value from closure to avoid additional parsing.
+                closure = (CounterClosure) iter.done();
+                counterOperation = closure.getCounterOperation();
+            } else {
+                // Have to parse FetchAddRequest from this user log.
+                final ByteBuffer data = iter.getData();
+                try {
+                    counterOperation = SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize(
+                        data.array(), CounterOperation.class.getName());
+                } catch (final CodecException e) {
+                    LOG.error("Fail to decode IncrementAndGetRequest", e);
+                }
+            }
+            if (counterOperation != null) {
+                switch (counterOperation.getOp()) {
+                    case GET:
+                        current = this.value.get();
+                        LOG.info("Get value={} at logIndex={}", current, iter.getIndex());
+                        break;
+                    case INCREMENT:
+                        final long delta = counterOperation.getDelta();
+                        final long prev = this.value.get();
+                        current = this.value.addAndGet(delta);
+                        LOG.info("Added value={} by delta={} at logIndex={}", prev, delta, iter.getIndex());
+                        break;
+                }
+
+                if (closure != null) {
+                    closure.success(current);
+                    closure.run(Status.OK());
+                }
+            }
+            iter.next();
+        }
+    }
+
+    @Override
+    public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+        final long currVal = this.value.get();
+        Utils.runInThread(() -> {
+            final CounterSnapshotFile snapshot = new CounterSnapshotFile(writer.getPath() + File.separator + "data");
+            if (snapshot.save(currVal)) {
+                if (writer.addFile("data")) {
+                    done.run(Status.OK());
+                } else {
+                    done.run(new Status(RaftError.EIO, "Fail to add file to writer"));
+                }
+            } else {
+                done.run(new Status(RaftError.EIO, "Fail to save counter snapshot %s", snapshot.getPath()));
+            }
+        });
+    }
+
+    @Override
+    public void onError(final RaftException e) {
+        LOG.error("Raft error: {}", e, e);
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        if (isLeader()) {
+            LOG.warn("Leader is not supposed to load snapshot");
+            return false;
+        }
+        if (reader.getFileMeta("data") == null) {
+            LOG.error("Fail to find data file in {}", reader.getPath());
+            return false;
+        }
+        final CounterSnapshotFile snapshot = new CounterSnapshotFile(reader.getPath() + File.separator + "data");
+        try {
+            this.value.set(snapshot.load());
+            return true;
+        } catch (final IOException e) {
+            LOG.error("Fail to load snapshot from {}", snapshot.getPath());
+            return false;
+        }
+
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        this.leaderTerm.set(term);
+        super.onLeaderStart(term);
+
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        this.leaderTerm.set(-1);
+        super.onLeaderStop(status);
+    }
+
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterGrpcHelper.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterGrpcHelper.java
new file mode 100644
index 0000000..b9d730b
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterGrpcHelper.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter.rpc;
+
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.util.concurrent.TimeUnit;
+
+public class CounterGrpcHelper {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CounterGrpcHelper.class);
+
+    public static RpcServer     rpcServer;
+
+    public static void initGRpc() {
+        if ("com.alipay.sofa.jraft.rpc.impl.GrpcRaftRpcFactory".equals(RpcFactoryHelper.rpcFactory().getClass()
+            .getName())) {
+            RpcFactoryHelper.rpcFactory().registerProtobufSerializer(CounterOutter.GetValueRequest.class.getName(),
+                CounterOutter.GetValueRequest.getDefaultInstance());
+            RpcFactoryHelper.rpcFactory().registerProtobufSerializer(
+                CounterOutter.IncrementAndGetRequest.class.getName(),
+                CounterOutter.IncrementAndGetRequest.getDefaultInstance());
+            RpcFactoryHelper.rpcFactory().registerProtobufSerializer(CounterOutter.ValueResponse.class.getName(),
+                CounterOutter.ValueResponse.getDefaultInstance());
+
+            try {
+                Class clazz = Class.forName("com.alipay.sofa.jraft.rpc.impl.MarshallerHelper");
+                Method registerRespInstance = clazz.getMethod("registerRespInstance", String.class, Message.class);
+                registerRespInstance.invoke(null, CounterOutter.GetValueRequest.class.getName(),
+                    CounterOutter.ValueResponse.getDefaultInstance());
+                registerRespInstance.invoke(null, CounterOutter.IncrementAndGetRequest.class.getName(),
+                    CounterOutter.ValueResponse.getDefaultInstance());
+            } catch (Exception e) {
+                LOG.error("Failed to init grpc server", e);
+            }
+        }
+    }
+
+    public static void setRpcServer(RpcServer rpcServer) {
+        CounterGrpcHelper.rpcServer = rpcServer;
+    }
+
+    public static void blockUntilShutdown() {
+        if (rpcServer == null) {
+            return;
+        }
+        if ("com.alipay.sofa.jraft.rpc.impl.GrpcRaftRpcFactory".equals(RpcFactoryHelper.rpcFactory().getClass()
+            .getName())) {
+            try {
+                Method getServer = rpcServer.getClass().getMethod("getServer");
+                Object grpcServer = getServer.invoke(rpcServer);
+
+                Method shutdown = grpcServer.getClass().getMethod("shutdown");
+                Method awaitTerminationLimit = grpcServer.getClass().getMethod("awaitTermination", long.class,
+                    TimeUnit.class);
+
+                Runtime.getRuntime().addShutdownHook(new Thread() {
+                    @Override
+                    public void run() {
+                        try {
+                            shutdown.invoke(grpcServer);
+                            awaitTerminationLimit.invoke(grpcServer, 30, TimeUnit.SECONDS);
+                        } catch (Exception e) {
+                            // Use stderr here since the logger may have been reset by its JVM shutdown hook.
+                            e.printStackTrace(System.err);
+                        }
+                    }
+                });
+                Method awaitTermination = grpcServer.getClass().getMethod("awaitTermination");
+                awaitTermination.invoke(grpcServer);
+            } catch (Exception e) {
+                LOG.error("Failed to block grpc server", e);
+            }
+        }
+    }
+
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterOutter.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterOutter.java
new file mode 100644
index 0000000..4758f05
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/CounterOutter.java
@@ -0,0 +1,1947 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+// Generated by the protocol buffer compiler.  DO NOT EDIT!
+// source: counter.proto
+
+package com.alipay.sofa.jraft.example.counter.rpc;
+
+public final class CounterOutter {
+    private CounterOutter() {
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistryLite registry) {
+    }
+
+    public static void registerAllExtensions(com.google.protobuf.ExtensionRegistry registry) {
+        registerAllExtensions((com.google.protobuf.ExtensionRegistryLite) registry);
+    }
+
+    public interface GetValueRequestOrBuilder extends
+    // @@protoc_insertion_point(interface_extends:jraft.GetValueRequest)
+                                             com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * required bool readOnlySafe = 1;
+         */
+        boolean hasReadOnlySafe();
+
+        /**
+         * required bool readOnlySafe = 1;
+         */
+        boolean getReadOnlySafe();
+    }
+
+    /**
+     * Protobuf type {@code jraft.GetValueRequest}
+     */
+    public static final class GetValueRequest extends com.google.protobuf.GeneratedMessageV3 implements
+    // @@protoc_insertion_point(message_implements:jraft.GetValueRequest)
+                                                                                            GetValueRequestOrBuilder {
+        private static final long serialVersionUID = 0L;
+
+        // Use GetValueRequest.newBuilder() to construct.
+        private GetValueRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+            super(builder);
+        }
+
+        private GetValueRequest() {
+            readOnlySafe_ = false;
+        }
+
+        @Override
+        public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
+            return this.unknownFields;
+        }
+
+        private GetValueRequest(com.google.protobuf.CodedInputStream input,
+                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                            throws com.google.protobuf.InvalidProtocolBufferException {
+            this();
+            if (extensionRegistry == null) {
+                throw new NullPointerException();
+            }
+            int mutable_bitField0_ = 0;
+            com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet
+                .newBuilder();
+            try {
+                boolean done = false;
+                while (!done) {
+                    int tag = input.readTag();
+                    switch (tag) {
+                        case 0:
+                            done = true;
+                            break;
+                        default: {
+                            if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
+                                done = true;
+                            }
+                            break;
+                        }
+                        case 8: {
+                            bitField0_ |= 0x00000001;
+                            readOnlySafe_ = input.readBool();
+                            break;
+                        }
+                    }
+                }
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(this);
+            } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this);
+            } finally {
+                this.unknownFields = unknownFields.build();
+                makeExtensionsImmutable();
+            }
+        }
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+            return CounterOutter.internal_static_jraft_GetValueRequest_descriptor;
+        }
+
+        protected FieldAccessorTable internalGetFieldAccessorTable() {
+            return CounterOutter.internal_static_jraft_GetValueRequest_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(GetValueRequest.class, Builder.class);
+        }
+
+        private int             bitField0_;
+        public static final int READONLYSAFE_FIELD_NUMBER = 1;
+        private boolean         readOnlySafe_;
+
+        /**
+         * required bool readOnlySafe = 1;
+         */
+        public boolean hasReadOnlySafe() {
+            return ((bitField0_ & 0x00000001) == 0x00000001);
+        }
+
+        /**
+         * required bool readOnlySafe = 1;
+         */
+        public boolean getReadOnlySafe() {
+            return readOnlySafe_;
+        }
+
+        private byte memoizedIsInitialized = -1;
+
+        public final boolean isInitialized() {
+            byte isInitialized = memoizedIsInitialized;
+            if (isInitialized == 1)
+                return true;
+            if (isInitialized == 0)
+                return false;
+
+            if (!hasReadOnlySafe()) {
+                memoizedIsInitialized = 0;
+                return false;
+            }
+            memoizedIsInitialized = 1;
+            return true;
+        }
+
+        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                output.writeBool(1, readOnlySafe_);
+            }
+            unknownFields.writeTo(output);
+        }
+
+        public int getSerializedSize() {
+            int size = memoizedSize;
+            if (size != -1)
+                return size;
+
+            size = 0;
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                size += com.google.protobuf.CodedOutputStream.computeBoolSize(1, readOnlySafe_);
+            }
+            size += unknownFields.getSerializedSize();
+            memoizedSize = size;
+            return size;
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (obj == this) {
+                return true;
+            }
+            if (!(obj instanceof GetValueRequest)) {
+                return super.equals(obj);
+            }
+            GetValueRequest other = (GetValueRequest) obj;
+
+            boolean result = true;
+            result = result && (hasReadOnlySafe() == other.hasReadOnlySafe());
+            if (hasReadOnlySafe()) {
+                result = result && (getReadOnlySafe() == other.getReadOnlySafe());
+            }
+            result = result && unknownFields.equals(other.unknownFields);
+            return result;
+        }
+
+        @Override
+        public int hashCode() {
+            if (memoizedHashCode != 0) {
+                return memoizedHashCode;
+            }
+            int hash = 41;
+            hash = (19 * hash) + getDescriptor().hashCode();
+            if (hasReadOnlySafe()) {
+                hash = (37 * hash) + READONLYSAFE_FIELD_NUMBER;
+                hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getReadOnlySafe());
+            }
+            hash = (29 * hash) + unknownFields.hashCode();
+            memoizedHashCode = hash;
+            return hash;
+        }
+
+        public static GetValueRequest parseFrom(java.nio.ByteBuffer data)
+                                                                         throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static GetValueRequest parseFrom(java.nio.ByteBuffer data,
+                                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                            throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static GetValueRequest parseFrom(com.google.protobuf.ByteString data)
+                                                                                    throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static GetValueRequest parseFrom(com.google.protobuf.ByteString data,
+                                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                            throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static GetValueRequest parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static GetValueRequest parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                         throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static GetValueRequest parseFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static GetValueRequest parseFrom(java.io.InputStream input,
+                                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                            throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public static GetValueRequest parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+        }
+
+        public static GetValueRequest parseDelimitedFrom(java.io.InputStream input,
+                                                         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                     throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input,
+                extensionRegistry);
+        }
+
+        public static GetValueRequest parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static GetValueRequest parseFrom(com.google.protobuf.CodedInputStream input,
+                                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                            throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public Builder newBuilderForType() {
+            return newBuilder();
+        }
+
+        public static Builder newBuilder() {
+            return DEFAULT_INSTANCE.toBuilder();
+        }
+
+        public static Builder newBuilder(GetValueRequest prototype) {
+            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+        public Builder toBuilder() {
+            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+        @Override
+        protected Builder newBuilderForType(BuilderParent parent) {
+            Builder builder = new Builder(parent);
+            return builder;
+        }
+
+        /**
+         * Protobuf type {@code jraft.GetValueRequest}
+         */
+        public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder implements
+        // @@protoc_insertion_point(builder_implements:jraft.GetValueRequest)
+                                                                                                         GetValueRequestOrBuilder {
+            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+                return CounterOutter.internal_static_jraft_GetValueRequest_descriptor;
+            }
+
+            protected FieldAccessorTable internalGetFieldAccessorTable() {
+                return CounterOutter.internal_static_jraft_GetValueRequest_fieldAccessorTable
+                    .ensureFieldAccessorsInitialized(GetValueRequest.class, Builder.class);
+            }
+
+            // Construct using com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.GetValueRequest.newBuilder()
+            private Builder() {
+                maybeForceBuilderInitialization();
+            }
+
+            private Builder(BuilderParent parent) {
+                super(parent);
+                maybeForceBuilderInitialization();
+            }
+
+            private void maybeForceBuilderInitialization() {
+                if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {
+                }
+            }
+
+            public Builder clear() {
+                super.clear();
+                readOnlySafe_ = false;
+                bitField0_ = (bitField0_ & ~0x00000001);
+                return this;
+            }
+
+            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+                return CounterOutter.internal_static_jraft_GetValueRequest_descriptor;
+            }
+
+            public GetValueRequest getDefaultInstanceForType() {
+                return GetValueRequest.getDefaultInstance();
+            }
+
+            public GetValueRequest build() {
+                GetValueRequest result = buildPartial();
+                if (!result.isInitialized()) {
+                    throw newUninitializedMessageException(result);
+                }
+                return result;
+            }
+
+            public GetValueRequest buildPartial() {
+                GetValueRequest result = new GetValueRequest(this);
+                int from_bitField0_ = bitField0_;
+                int to_bitField0_ = 0;
+                if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+                    to_bitField0_ |= 0x00000001;
+                }
+                result.readOnlySafe_ = readOnlySafe_;
+                result.bitField0_ = to_bitField0_;
+                onBuilt();
+                return result;
+            }
+
+            public Builder clone() {
+                return (Builder) super.clone();
+            }
+
+            public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.setField(field, value);
+            }
+
+            public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+                return (Builder) super.clearField(field);
+            }
+
+            public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+                return (Builder) super.clearOneof(oneof);
+            }
+
+            public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index,
+                                            Object value) {
+                return (Builder) super.setRepeatedField(field, index, value);
+            }
+
+            public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.addRepeatedField(field, value);
+            }
+
+            public Builder mergeFrom(com.google.protobuf.Message other) {
+                if (other instanceof GetValueRequest) {
+                    return mergeFrom((GetValueRequest) other);
+                } else {
+                    super.mergeFrom(other);
+                    return this;
+                }
+            }
+
+            public Builder mergeFrom(GetValueRequest other) {
+                if (other == GetValueRequest.getDefaultInstance())
+                    return this;
+                if (other.hasReadOnlySafe()) {
+                    setReadOnlySafe(other.getReadOnlySafe());
+                }
+                this.mergeUnknownFields(other.unknownFields);
+                onChanged();
+                return this;
+            }
+
+            public final boolean isInitialized() {
+                if (!hasReadOnlySafe()) {
+                    return false;
+                }
+                return true;
+            }
+
+            public Builder mergeFrom(com.google.protobuf.CodedInputStream input,
+                                     com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                 throws java.io.IOException {
+                GetValueRequest parsedMessage = null;
+                try {
+                    parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    parsedMessage = (GetValueRequest) e.getUnfinishedMessage();
+                    throw e.unwrapIOException();
+                } finally {
+                    if (parsedMessage != null) {
+                        mergeFrom(parsedMessage);
+                    }
+                }
+                return this;
+            }
+
+            private int     bitField0_;
+
+            private boolean readOnlySafe_;
+
+            /**
+             * required bool readOnlySafe = 1;
+             */
+            public boolean hasReadOnlySafe() {
+                return ((bitField0_ & 0x00000001) == 0x00000001);
+            }
+
+            /**
+             * required bool readOnlySafe = 1;
+             */
+            public boolean getReadOnlySafe() {
+                return readOnlySafe_;
+            }
+
+            /**
+             * required bool readOnlySafe = 1;
+             */
+            public Builder setReadOnlySafe(boolean value) {
+                bitField0_ |= 0x00000001;
+                readOnlySafe_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * required bool readOnlySafe = 1;
+             */
+            public Builder clearReadOnlySafe() {
+                bitField0_ = (bitField0_ & ~0x00000001);
+                readOnlySafe_ = false;
+                onChanged();
+                return this;
+            }
+
+            public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.setUnknownFields(unknownFields);
+            }
+
+            public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.mergeUnknownFields(unknownFields);
+            }
+
+            // @@protoc_insertion_point(builder_scope:jraft.GetValueRequest)
+        }
+
+        // @@protoc_insertion_point(class_scope:jraft.GetValueRequest)
+        private static final GetValueRequest DEFAULT_INSTANCE;
+        static {
+            DEFAULT_INSTANCE = new GetValueRequest();
+        }
+
+        public static GetValueRequest getDefaultInstance() {
+            return DEFAULT_INSTANCE;
+        }
+
+        @Deprecated
+        public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() {
+                                                                                   public GetValueRequest parsePartialFrom(com.google.protobuf.CodedInputStream input,
+                                                                                                                           com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                                                                                       throws com.google.protobuf.InvalidProtocolBufferException {
+                                                                                       return new GetValueRequest(
+                                                                                           input, extensionRegistry);
+                                                                                   }
+                                                                               };
+
+        public static com.google.protobuf.Parser parser() {
+            return PARSER;
+        }
+
+        @Override
+        public com.google.protobuf.Parser getParserForType() {
+            return PARSER;
+        }
+
+        public GetValueRequest getDefaultInstanceForType() {
+            return DEFAULT_INSTANCE;
+        }
+
+    }
+
+    public interface IncrementAndGetRequestOrBuilder extends
+    // @@protoc_insertion_point(interface_extends:jraft.IncrementAndGetRequest)
+                                                    com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * required int64 delta = 1;
+         */
+        boolean hasDelta();
+
+        /**
+         * required int64 delta = 1;
+         */
+        long getDelta();
+    }
+
+    /**
+     * Protobuf type {@code jraft.IncrementAndGetRequest}
+     */
+    public static final class IncrementAndGetRequest extends com.google.protobuf.GeneratedMessageV3 implements
+    // @@protoc_insertion_point(message_implements:jraft.IncrementAndGetRequest)
+                                                                                                   IncrementAndGetRequestOrBuilder {
+        private static final long serialVersionUID = 0L;
+
+        // Use IncrementAndGetRequest.newBuilder() to construct.
+        private IncrementAndGetRequest(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+            super(builder);
+        }
+
+        private IncrementAndGetRequest() {
+            delta_ = 0L;
+        }
+
+        @Override
+        public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
+            return this.unknownFields;
+        }
+
+        private IncrementAndGetRequest(com.google.protobuf.CodedInputStream input,
+                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+            this();
+            if (extensionRegistry == null) {
+                throw new NullPointerException();
+            }
+            int mutable_bitField0_ = 0;
+            com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet
+                .newBuilder();
+            try {
+                boolean done = false;
+                while (!done) {
+                    int tag = input.readTag();
+                    switch (tag) {
+                        case 0:
+                            done = true;
+                            break;
+                        default: {
+                            if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
+                                done = true;
+                            }
+                            break;
+                        }
+                        case 8: {
+                            bitField0_ |= 0x00000001;
+                            delta_ = input.readInt64();
+                            break;
+                        }
+                    }
+                }
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(this);
+            } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this);
+            } finally {
+                this.unknownFields = unknownFields.build();
+                makeExtensionsImmutable();
+            }
+        }
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+            return CounterOutter.internal_static_jraft_IncrementAndGetRequest_descriptor;
+        }
+
+        protected FieldAccessorTable internalGetFieldAccessorTable() {
+            return CounterOutter.internal_static_jraft_IncrementAndGetRequest_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(IncrementAndGetRequest.class, Builder.class);
+        }
+
+        private int             bitField0_;
+        public static final int DELTA_FIELD_NUMBER = 1;
+        private long            delta_;
+
+        /**
+         * required int64 delta = 1;
+         */
+        public boolean hasDelta() {
+            return ((bitField0_ & 0x00000001) == 0x00000001);
+        }
+
+        /**
+         * required int64 delta = 1;
+         */
+        public long getDelta() {
+            return delta_;
+        }
+
+        private byte memoizedIsInitialized = -1;
+
+        public final boolean isInitialized() {
+            byte isInitialized = memoizedIsInitialized;
+            if (isInitialized == 1)
+                return true;
+            if (isInitialized == 0)
+                return false;
+
+            if (!hasDelta()) {
+                memoizedIsInitialized = 0;
+                return false;
+            }
+            memoizedIsInitialized = 1;
+            return true;
+        }
+
+        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                output.writeInt64(1, delta_);
+            }
+            unknownFields.writeTo(output);
+        }
+
+        public int getSerializedSize() {
+            int size = memoizedSize;
+            if (size != -1)
+                return size;
+
+            size = 0;
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, delta_);
+            }
+            size += unknownFields.getSerializedSize();
+            memoizedSize = size;
+            return size;
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (obj == this) {
+                return true;
+            }
+            if (!(obj instanceof IncrementAndGetRequest)) {
+                return super.equals(obj);
+            }
+            IncrementAndGetRequest other = (IncrementAndGetRequest) obj;
+
+            boolean result = true;
+            result = result && (hasDelta() == other.hasDelta());
+            if (hasDelta()) {
+                result = result && (getDelta() == other.getDelta());
+            }
+            result = result && unknownFields.equals(other.unknownFields);
+            return result;
+        }
+
+        @Override
+        public int hashCode() {
+            if (memoizedHashCode != 0) {
+                return memoizedHashCode;
+            }
+            int hash = 41;
+            hash = (19 * hash) + getDescriptor().hashCode();
+            if (hasDelta()) {
+                hash = (37 * hash) + DELTA_FIELD_NUMBER;
+                hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getDelta());
+            }
+            hash = (29 * hash) + unknownFields.hashCode();
+            memoizedHashCode = hash;
+            return hash;
+        }
+
+        public static IncrementAndGetRequest parseFrom(java.nio.ByteBuffer data)
+                                                                                throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static IncrementAndGetRequest parseFrom(java.nio.ByteBuffer data,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static IncrementAndGetRequest parseFrom(com.google.protobuf.ByteString data)
+                                                                                           throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static IncrementAndGetRequest parseFrom(com.google.protobuf.ByteString data,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static IncrementAndGetRequest parseFrom(byte[] data)
+                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static IncrementAndGetRequest parseFrom(byte[] data,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static IncrementAndGetRequest parseFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static IncrementAndGetRequest parseFrom(java.io.InputStream input,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public static IncrementAndGetRequest parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+        }
+
+        public static IncrementAndGetRequest parseDelimitedFrom(java.io.InputStream input,
+                                                                com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                            throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input,
+                extensionRegistry);
+        }
+
+        public static IncrementAndGetRequest parseFrom(com.google.protobuf.CodedInputStream input)
+                                                                                                  throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static IncrementAndGetRequest parseFrom(com.google.protobuf.CodedInputStream input,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public Builder newBuilderForType() {
+            return newBuilder();
+        }
+
+        public static Builder newBuilder() {
+            return DEFAULT_INSTANCE.toBuilder();
+        }
+
+        public static Builder newBuilder(IncrementAndGetRequest prototype) {
+            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+        public Builder toBuilder() {
+            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+        @Override
+        protected Builder newBuilderForType(BuilderParent parent) {
+            Builder builder = new Builder(parent);
+            return builder;
+        }
+
+        /**
+         * Protobuf type {@code jraft.IncrementAndGetRequest}
+         */
+        public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder implements
+        // @@protoc_insertion_point(builder_implements:jraft.IncrementAndGetRequest)
+                                                                                                         IncrementAndGetRequestOrBuilder {
+            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+                return CounterOutter.internal_static_jraft_IncrementAndGetRequest_descriptor;
+            }
+
+            protected FieldAccessorTable internalGetFieldAccessorTable() {
+                return CounterOutter.internal_static_jraft_IncrementAndGetRequest_fieldAccessorTable
+                    .ensureFieldAccessorsInitialized(IncrementAndGetRequest.class, Builder.class);
+            }
+
+            // Construct using com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.IncrementAndGetRequest.newBuilder()
+            private Builder() {
+                maybeForceBuilderInitialization();
+            }
+
+            private Builder(BuilderParent parent) {
+                super(parent);
+                maybeForceBuilderInitialization();
+            }
+
+            private void maybeForceBuilderInitialization() {
+                if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {
+                }
+            }
+
+            public Builder clear() {
+                super.clear();
+                delta_ = 0L;
+                bitField0_ = (bitField0_ & ~0x00000001);
+                return this;
+            }
+
+            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+                return CounterOutter.internal_static_jraft_IncrementAndGetRequest_descriptor;
+            }
+
+            public IncrementAndGetRequest getDefaultInstanceForType() {
+                return IncrementAndGetRequest.getDefaultInstance();
+            }
+
+            public IncrementAndGetRequest build() {
+                IncrementAndGetRequest result = buildPartial();
+                if (!result.isInitialized()) {
+                    throw newUninitializedMessageException(result);
+                }
+                return result;
+            }
+
+            public IncrementAndGetRequest buildPartial() {
+                IncrementAndGetRequest result = new IncrementAndGetRequest(this);
+                int from_bitField0_ = bitField0_;
+                int to_bitField0_ = 0;
+                if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+                    to_bitField0_ |= 0x00000001;
+                }
+                result.delta_ = delta_;
+                result.bitField0_ = to_bitField0_;
+                onBuilt();
+                return result;
+            }
+
+            public Builder clone() {
+                return (Builder) super.clone();
+            }
+
+            public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.setField(field, value);
+            }
+
+            public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+                return (Builder) super.clearField(field);
+            }
+
+            public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+                return (Builder) super.clearOneof(oneof);
+            }
+
+            public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index,
+                                            Object value) {
+                return (Builder) super.setRepeatedField(field, index, value);
+            }
+
+            public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.addRepeatedField(field, value);
+            }
+
+            public Builder mergeFrom(com.google.protobuf.Message other) {
+                if (other instanceof IncrementAndGetRequest) {
+                    return mergeFrom((IncrementAndGetRequest) other);
+                } else {
+                    super.mergeFrom(other);
+                    return this;
+                }
+            }
+
+            public Builder mergeFrom(IncrementAndGetRequest other) {
+                if (other == IncrementAndGetRequest.getDefaultInstance())
+                    return this;
+                if (other.hasDelta()) {
+                    setDelta(other.getDelta());
+                }
+                this.mergeUnknownFields(other.unknownFields);
+                onChanged();
+                return this;
+            }
+
+            public final boolean isInitialized() {
+                if (!hasDelta()) {
+                    return false;
+                }
+                return true;
+            }
+
+            public Builder mergeFrom(com.google.protobuf.CodedInputStream input,
+                                     com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                 throws java.io.IOException {
+                IncrementAndGetRequest parsedMessage = null;
+                try {
+                    parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    parsedMessage = (IncrementAndGetRequest) e.getUnfinishedMessage();
+                    throw e.unwrapIOException();
+                } finally {
+                    if (parsedMessage != null) {
+                        mergeFrom(parsedMessage);
+                    }
+                }
+                return this;
+            }
+
+            private int  bitField0_;
+
+            private long delta_;
+
+            /**
+             * required int64 delta = 1;
+             */
+            public boolean hasDelta() {
+                return ((bitField0_ & 0x00000001) == 0x00000001);
+            }
+
+            /**
+             * required int64 delta = 1;
+             */
+            public long getDelta() {
+                return delta_;
+            }
+
+            /**
+             * required int64 delta = 1;
+             */
+            public Builder setDelta(long value) {
+                bitField0_ |= 0x00000001;
+                delta_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * required int64 delta = 1;
+             */
+            public Builder clearDelta() {
+                bitField0_ = (bitField0_ & ~0x00000001);
+                delta_ = 0L;
+                onChanged();
+                return this;
+            }
+
+            public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.setUnknownFields(unknownFields);
+            }
+
+            public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.mergeUnknownFields(unknownFields);
+            }
+
+            // @@protoc_insertion_point(builder_scope:jraft.IncrementAndGetRequest)
+        }
+
+        // @@protoc_insertion_point(class_scope:jraft.IncrementAndGetRequest)
+        private static final IncrementAndGetRequest DEFAULT_INSTANCE;
+        static {
+            DEFAULT_INSTANCE = new IncrementAndGetRequest();
+        }
+
+        public static IncrementAndGetRequest getDefaultInstance() {
+            return DEFAULT_INSTANCE;
+        }
+
+        @Deprecated
+        public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() {
+                                                                                          public IncrementAndGetRequest parsePartialFrom(com.google.protobuf.CodedInputStream input,
+                                                                                                                                         com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                                                                                                     throws com.google.protobuf.InvalidProtocolBufferException {
+                                                                                              return new IncrementAndGetRequest(
+                                                                                                  input,
+                                                                                                  extensionRegistry);
+                                                                                          }
+                                                                                      };
+
+        public static com.google.protobuf.Parser parser() {
+            return PARSER;
+        }
+
+        @Override
+        public com.google.protobuf.Parser getParserForType() {
+            return PARSER;
+        }
+
+        public IncrementAndGetRequest getDefaultInstanceForType() {
+            return DEFAULT_INSTANCE;
+        }
+
+    }
+
+    public interface ValueResponseOrBuilder extends
+    // @@protoc_insertion_point(interface_extends:jraft.ValueResponse)
+                                           com.google.protobuf.MessageOrBuilder {
+
+        /**
+         * required int64 value = 1;
+         */
+        boolean hasValue();
+
+        /**
+         * required int64 value = 1;
+         */
+        long getValue();
+
+        /**
+         * required bool success = 2;
+         */
+        boolean hasSuccess();
+
+        /**
+         * required bool success = 2;
+         */
+        boolean getSuccess();
+
+        /**
+         * optional string redirect = 3;
+         */
+        boolean hasRedirect();
+
+        /**
+         * optional string redirect = 3;
+         */
+        String getRedirect();
+
+        /**
+         * optional string redirect = 3;
+         */
+        com.google.protobuf.ByteString getRedirectBytes();
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        boolean hasErrorMsg();
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        String getErrorMsg();
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        com.google.protobuf.ByteString getErrorMsgBytes();
+    }
+
+    /**
+     * Protobuf type {@code jraft.ValueResponse}
+     */
+    public static final class ValueResponse extends com.google.protobuf.GeneratedMessageV3 implements
+    // @@protoc_insertion_point(message_implements:jraft.ValueResponse)
+                                                                                          ValueResponseOrBuilder {
+        private static final long serialVersionUID = 0L;
+
+        // Use ValueResponse.newBuilder() to construct.
+        private ValueResponse(com.google.protobuf.GeneratedMessageV3.Builder builder) {
+            super(builder);
+        }
+
+        private ValueResponse() {
+            value_ = 0L;
+            success_ = false;
+            redirect_ = "";
+            errorMsg_ = "";
+        }
+
+        @Override
+        public final com.google.protobuf.UnknownFieldSet getUnknownFields() {
+            return this.unknownFields;
+        }
+
+        private ValueResponse(com.google.protobuf.CodedInputStream input,
+                              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                          throws com.google.protobuf.InvalidProtocolBufferException {
+            this();
+            if (extensionRegistry == null) {
+                throw new NullPointerException();
+            }
+            int mutable_bitField0_ = 0;
+            com.google.protobuf.UnknownFieldSet.Builder unknownFields = com.google.protobuf.UnknownFieldSet
+                .newBuilder();
+            try {
+                boolean done = false;
+                while (!done) {
+                    int tag = input.readTag();
+                    switch (tag) {
+                        case 0:
+                            done = true;
+                            break;
+                        default: {
+                            if (!parseUnknownField(input, unknownFields, extensionRegistry, tag)) {
+                                done = true;
+                            }
+                            break;
+                        }
+                        case 8: {
+                            bitField0_ |= 0x00000001;
+                            value_ = input.readInt64();
+                            break;
+                        }
+                        case 16: {
+                            bitField0_ |= 0x00000002;
+                            success_ = input.readBool();
+                            break;
+                        }
+                        case 26: {
+                            com.google.protobuf.ByteString bs = input.readBytes();
+                            bitField0_ |= 0x00000004;
+                            redirect_ = bs;
+                            break;
+                        }
+                        case 34: {
+                            com.google.protobuf.ByteString bs = input.readBytes();
+                            bitField0_ |= 0x00000008;
+                            errorMsg_ = bs;
+                            break;
+                        }
+                    }
+                }
+            } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                throw e.setUnfinishedMessage(this);
+            } catch (java.io.IOException e) {
+                throw new com.google.protobuf.InvalidProtocolBufferException(e).setUnfinishedMessage(this);
+            } finally {
+                this.unknownFields = unknownFields.build();
+                makeExtensionsImmutable();
+            }
+        }
+
+        public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+            return CounterOutter.internal_static_jraft_ValueResponse_descriptor;
+        }
+
+        protected FieldAccessorTable internalGetFieldAccessorTable() {
+            return CounterOutter.internal_static_jraft_ValueResponse_fieldAccessorTable
+                .ensureFieldAccessorsInitialized(ValueResponse.class, Builder.class);
+        }
+
+        private int             bitField0_;
+        public static final int VALUE_FIELD_NUMBER = 1;
+        private long            value_;
+
+        /**
+         * required int64 value = 1;
+         */
+        public boolean hasValue() {
+            return ((bitField0_ & 0x00000001) == 0x00000001);
+        }
+
+        /**
+         * required int64 value = 1;
+         */
+        public long getValue() {
+            return value_;
+        }
+
+        public static final int SUCCESS_FIELD_NUMBER = 2;
+        private boolean         success_;
+
+        /**
+         * required bool success = 2;
+         */
+        public boolean hasSuccess() {
+            return ((bitField0_ & 0x00000002) == 0x00000002);
+        }
+
+        /**
+         * required bool success = 2;
+         */
+        public boolean getSuccess() {
+            return success_;
+        }
+
+        public static final int REDIRECT_FIELD_NUMBER = 3;
+        private volatile Object redirect_;
+
+        /**
+         * optional string redirect = 3;
+         */
+        public boolean hasRedirect() {
+            return ((bitField0_ & 0x00000004) == 0x00000004);
+        }
+
+        /**
+         * optional string redirect = 3;
+         */
+        public String getRedirect() {
+            Object ref = redirect_;
+            if (ref instanceof String) {
+                return (String) ref;
+            } else {
+                com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+                String s = bs.toStringUtf8();
+                if (bs.isValidUtf8()) {
+                    redirect_ = s;
+                }
+                return s;
+            }
+        }
+
+        /**
+         * optional string redirect = 3;
+         */
+        public com.google.protobuf.ByteString getRedirectBytes() {
+            Object ref = redirect_;
+            if (ref instanceof String) {
+                com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+                redirect_ = b;
+                return b;
+            } else {
+                return (com.google.protobuf.ByteString) ref;
+            }
+        }
+
+        public static final int ERRORMSG_FIELD_NUMBER = 4;
+        private volatile Object errorMsg_;
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        public boolean hasErrorMsg() {
+            return ((bitField0_ & 0x00000008) == 0x00000008);
+        }
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        public String getErrorMsg() {
+            Object ref = errorMsg_;
+            if (ref instanceof String) {
+                return (String) ref;
+            } else {
+                com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+                String s = bs.toStringUtf8();
+                if (bs.isValidUtf8()) {
+                    errorMsg_ = s;
+                }
+                return s;
+            }
+        }
+
+        /**
+         * optional string errorMsg = 4;
+         */
+        public com.google.protobuf.ByteString getErrorMsgBytes() {
+            Object ref = errorMsg_;
+            if (ref instanceof String) {
+                com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+                errorMsg_ = b;
+                return b;
+            } else {
+                return (com.google.protobuf.ByteString) ref;
+            }
+        }
+
+        private byte memoizedIsInitialized = -1;
+
+        public final boolean isInitialized() {
+            byte isInitialized = memoizedIsInitialized;
+            if (isInitialized == 1)
+                return true;
+            if (isInitialized == 0)
+                return false;
+
+            if (!hasValue()) {
+                memoizedIsInitialized = 0;
+                return false;
+            }
+            if (!hasSuccess()) {
+                memoizedIsInitialized = 0;
+                return false;
+            }
+            memoizedIsInitialized = 1;
+            return true;
+        }
+
+        public void writeTo(com.google.protobuf.CodedOutputStream output) throws java.io.IOException {
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                output.writeInt64(1, value_);
+            }
+            if (((bitField0_ & 0x00000002) == 0x00000002)) {
+                output.writeBool(2, success_);
+            }
+            if (((bitField0_ & 0x00000004) == 0x00000004)) {
+                com.google.protobuf.GeneratedMessageV3.writeString(output, 3, redirect_);
+            }
+            if (((bitField0_ & 0x00000008) == 0x00000008)) {
+                com.google.protobuf.GeneratedMessageV3.writeString(output, 4, errorMsg_);
+            }
+            unknownFields.writeTo(output);
+        }
+
+        public int getSerializedSize() {
+            int size = memoizedSize;
+            if (size != -1)
+                return size;
+
+            size = 0;
+            if (((bitField0_ & 0x00000001) == 0x00000001)) {
+                size += com.google.protobuf.CodedOutputStream.computeInt64Size(1, value_);
+            }
+            if (((bitField0_ & 0x00000002) == 0x00000002)) {
+                size += com.google.protobuf.CodedOutputStream.computeBoolSize(2, success_);
+            }
+            if (((bitField0_ & 0x00000004) == 0x00000004)) {
+                size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, redirect_);
+            }
+            if (((bitField0_ & 0x00000008) == 0x00000008)) {
+                size += com.google.protobuf.GeneratedMessageV3.computeStringSize(4, errorMsg_);
+            }
+            size += unknownFields.getSerializedSize();
+            memoizedSize = size;
+            return size;
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (obj == this) {
+                return true;
+            }
+            if (!(obj instanceof ValueResponse)) {
+                return super.equals(obj);
+            }
+            ValueResponse other = (ValueResponse) obj;
+
+            boolean result = true;
+            result = result && (hasValue() == other.hasValue());
+            if (hasValue()) {
+                result = result && (getValue() == other.getValue());
+            }
+            result = result && (hasSuccess() == other.hasSuccess());
+            if (hasSuccess()) {
+                result = result && (getSuccess() == other.getSuccess());
+            }
+            result = result && (hasRedirect() == other.hasRedirect());
+            if (hasRedirect()) {
+                result = result && getRedirect().equals(other.getRedirect());
+            }
+            result = result && (hasErrorMsg() == other.hasErrorMsg());
+            if (hasErrorMsg()) {
+                result = result && getErrorMsg().equals(other.getErrorMsg());
+            }
+            result = result && unknownFields.equals(other.unknownFields);
+            return result;
+        }
+
+        @Override
+        public int hashCode() {
+            if (memoizedHashCode != 0) {
+                return memoizedHashCode;
+            }
+            int hash = 41;
+            hash = (19 * hash) + getDescriptor().hashCode();
+            if (hasValue()) {
+                hash = (37 * hash) + VALUE_FIELD_NUMBER;
+                hash = (53 * hash) + com.google.protobuf.Internal.hashLong(getValue());
+            }
+            if (hasSuccess()) {
+                hash = (37 * hash) + SUCCESS_FIELD_NUMBER;
+                hash = (53 * hash) + com.google.protobuf.Internal.hashBoolean(getSuccess());
+            }
+            if (hasRedirect()) {
+                hash = (37 * hash) + REDIRECT_FIELD_NUMBER;
+                hash = (53 * hash) + getRedirect().hashCode();
+            }
+            if (hasErrorMsg()) {
+                hash = (37 * hash) + ERRORMSG_FIELD_NUMBER;
+                hash = (53 * hash) + getErrorMsg().hashCode();
+            }
+            hash = (29 * hash) + unknownFields.hashCode();
+            memoizedHashCode = hash;
+            return hash;
+        }
+
+        public static ValueResponse parseFrom(java.nio.ByteBuffer data)
+                                                                       throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static ValueResponse parseFrom(java.nio.ByteBuffer data,
+                                              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                          throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static ValueResponse parseFrom(com.google.protobuf.ByteString data)
+                                                                                  throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static ValueResponse parseFrom(com.google.protobuf.ByteString data,
+                                              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                          throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static ValueResponse parseFrom(byte[] data) throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data);
+        }
+
+        public static ValueResponse parseFrom(byte[] data, com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                       throws com.google.protobuf.InvalidProtocolBufferException {
+            return PARSER.parseFrom(data, extensionRegistry);
+        }
+
+        public static ValueResponse parseFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static ValueResponse parseFrom(java.io.InputStream input,
+                                              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                          throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public static ValueResponse parseDelimitedFrom(java.io.InputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input);
+        }
+
+        public static ValueResponse parseDelimitedFrom(java.io.InputStream input,
+                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                   throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseDelimitedWithIOException(PARSER, input,
+                extensionRegistry);
+        }
+
+        public static ValueResponse parseFrom(com.google.protobuf.CodedInputStream input) throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input);
+        }
+
+        public static ValueResponse parseFrom(com.google.protobuf.CodedInputStream input,
+                                              com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                          throws java.io.IOException {
+            return com.google.protobuf.GeneratedMessageV3.parseWithIOException(PARSER, input, extensionRegistry);
+        }
+
+        public Builder newBuilderForType() {
+            return newBuilder();
+        }
+
+        public static Builder newBuilder() {
+            return DEFAULT_INSTANCE.toBuilder();
+        }
+
+        public static Builder newBuilder(ValueResponse prototype) {
+            return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype);
+        }
+
+        public Builder toBuilder() {
+            return this == DEFAULT_INSTANCE ? new Builder() : new Builder().mergeFrom(this);
+        }
+
+        @Override
+        protected Builder newBuilderForType(BuilderParent parent) {
+            Builder builder = new Builder(parent);
+            return builder;
+        }
+
+        /**
+         * Protobuf type {@code jraft.ValueResponse}
+         */
+        public static final class Builder extends com.google.protobuf.GeneratedMessageV3.Builder implements
+        // @@protoc_insertion_point(builder_implements:jraft.ValueResponse)
+                                                                                                         ValueResponseOrBuilder {
+            public static final com.google.protobuf.Descriptors.Descriptor getDescriptor() {
+                return CounterOutter.internal_static_jraft_ValueResponse_descriptor;
+            }
+
+            protected FieldAccessorTable internalGetFieldAccessorTable() {
+                return CounterOutter.internal_static_jraft_ValueResponse_fieldAccessorTable
+                    .ensureFieldAccessorsInitialized(ValueResponse.class, Builder.class);
+            }
+
+            // Construct using com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.ValueResponse.newBuilder()
+            private Builder() {
+                maybeForceBuilderInitialization();
+            }
+
+            private Builder(BuilderParent parent) {
+                super(parent);
+                maybeForceBuilderInitialization();
+            }
+
+            private void maybeForceBuilderInitialization() {
+                if (com.google.protobuf.GeneratedMessageV3.alwaysUseFieldBuilders) {
+                }
+            }
+
+            public Builder clear() {
+                super.clear();
+                value_ = 0L;
+                bitField0_ = (bitField0_ & ~0x00000001);
+                success_ = false;
+                bitField0_ = (bitField0_ & ~0x00000002);
+                redirect_ = "";
+                bitField0_ = (bitField0_ & ~0x00000004);
+                errorMsg_ = "";
+                bitField0_ = (bitField0_ & ~0x00000008);
+                return this;
+            }
+
+            public com.google.protobuf.Descriptors.Descriptor getDescriptorForType() {
+                return CounterOutter.internal_static_jraft_ValueResponse_descriptor;
+            }
+
+            public ValueResponse getDefaultInstanceForType() {
+                return ValueResponse.getDefaultInstance();
+            }
+
+            public ValueResponse build() {
+                ValueResponse result = buildPartial();
+                if (!result.isInitialized()) {
+                    throw newUninitializedMessageException(result);
+                }
+                return result;
+            }
+
+            public ValueResponse buildPartial() {
+                ValueResponse result = new ValueResponse(this);
+                int from_bitField0_ = bitField0_;
+                int to_bitField0_ = 0;
+                if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+                    to_bitField0_ |= 0x00000001;
+                }
+                result.value_ = value_;
+                if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+                    to_bitField0_ |= 0x00000002;
+                }
+                result.success_ = success_;
+                if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+                    to_bitField0_ |= 0x00000004;
+                }
+                result.redirect_ = redirect_;
+                if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+                    to_bitField0_ |= 0x00000008;
+                }
+                result.errorMsg_ = errorMsg_;
+                result.bitField0_ = to_bitField0_;
+                onBuilt();
+                return result;
+            }
+
+            public Builder clone() {
+                return (Builder) super.clone();
+            }
+
+            public Builder setField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.setField(field, value);
+            }
+
+            public Builder clearField(com.google.protobuf.Descriptors.FieldDescriptor field) {
+                return (Builder) super.clearField(field);
+            }
+
+            public Builder clearOneof(com.google.protobuf.Descriptors.OneofDescriptor oneof) {
+                return (Builder) super.clearOneof(oneof);
+            }
+
+            public Builder setRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, int index,
+                                            Object value) {
+                return (Builder) super.setRepeatedField(field, index, value);
+            }
+
+            public Builder addRepeatedField(com.google.protobuf.Descriptors.FieldDescriptor field, Object value) {
+                return (Builder) super.addRepeatedField(field, value);
+            }
+
+            public Builder mergeFrom(com.google.protobuf.Message other) {
+                if (other instanceof ValueResponse) {
+                    return mergeFrom((ValueResponse) other);
+                } else {
+                    super.mergeFrom(other);
+                    return this;
+                }
+            }
+
+            public Builder mergeFrom(ValueResponse other) {
+                if (other == ValueResponse.getDefaultInstance())
+                    return this;
+                if (other.hasValue()) {
+                    setValue(other.getValue());
+                }
+                if (other.hasSuccess()) {
+                    setSuccess(other.getSuccess());
+                }
+                if (other.hasRedirect()) {
+                    bitField0_ |= 0x00000004;
+                    redirect_ = other.redirect_;
+                    onChanged();
+                }
+                if (other.hasErrorMsg()) {
+                    bitField0_ |= 0x00000008;
+                    errorMsg_ = other.errorMsg_;
+                    onChanged();
+                }
+                this.mergeUnknownFields(other.unknownFields);
+                onChanged();
+                return this;
+            }
+
+            public final boolean isInitialized() {
+                if (!hasValue()) {
+                    return false;
+                }
+                if (!hasSuccess()) {
+                    return false;
+                }
+                return true;
+            }
+
+            public Builder mergeFrom(com.google.protobuf.CodedInputStream input,
+                                     com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                 throws java.io.IOException {
+                ValueResponse parsedMessage = null;
+                try {
+                    parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+                } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+                    parsedMessage = (ValueResponse) e.getUnfinishedMessage();
+                    throw e.unwrapIOException();
+                } finally {
+                    if (parsedMessage != null) {
+                        mergeFrom(parsedMessage);
+                    }
+                }
+                return this;
+            }
+
+            private int  bitField0_;
+
+            private long value_;
+
+            /**
+             * required int64 value = 1;
+             */
+            public boolean hasValue() {
+                return ((bitField0_ & 0x00000001) == 0x00000001);
+            }
+
+            /**
+             * required int64 value = 1;
+             */
+            public long getValue() {
+                return value_;
+            }
+
+            /**
+             * required int64 value = 1;
+             */
+            public Builder setValue(long value) {
+                bitField0_ |= 0x00000001;
+                value_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * required int64 value = 1;
+             */
+            public Builder clearValue() {
+                bitField0_ = (bitField0_ & ~0x00000001);
+                value_ = 0L;
+                onChanged();
+                return this;
+            }
+
+            private boolean success_;
+
+            /**
+             * required bool success = 2;
+             */
+            public boolean hasSuccess() {
+                return ((bitField0_ & 0x00000002) == 0x00000002);
+            }
+
+            /**
+             * required bool success = 2;
+             */
+            public boolean getSuccess() {
+                return success_;
+            }
+
+            /**
+             * required bool success = 2;
+             */
+            public Builder setSuccess(boolean value) {
+                bitField0_ |= 0x00000002;
+                success_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * required bool success = 2;
+             */
+            public Builder clearSuccess() {
+                bitField0_ = (bitField0_ & ~0x00000002);
+                success_ = false;
+                onChanged();
+                return this;
+            }
+
+            private Object redirect_ = "";
+
+            /**
+             * optional string redirect = 3;
+             */
+            public boolean hasRedirect() {
+                return ((bitField0_ & 0x00000004) == 0x00000004);
+            }
+
+            /**
+             * optional string redirect = 3;
+             */
+            public String getRedirect() {
+                Object ref = redirect_;
+                if (!(ref instanceof String)) {
+                    com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+                    String s = bs.toStringUtf8();
+                    if (bs.isValidUtf8()) {
+                        redirect_ = s;
+                    }
+                    return s;
+                } else {
+                    return (String) ref;
+                }
+            }
+
+            /**
+             * optional string redirect = 3;
+             */
+            public com.google.protobuf.ByteString getRedirectBytes() {
+                Object ref = redirect_;
+                if (ref instanceof String) {
+                    com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+                    redirect_ = b;
+                    return b;
+                } else {
+                    return (com.google.protobuf.ByteString) ref;
+                }
+            }
+
+            /**
+             * optional string redirect = 3;
+             */
+            public Builder setRedirect(String value) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                bitField0_ |= 0x00000004;
+                redirect_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * optional string redirect = 3;
+             */
+            public Builder clearRedirect() {
+                bitField0_ = (bitField0_ & ~0x00000004);
+                redirect_ = getDefaultInstance().getRedirect();
+                onChanged();
+                return this;
+            }
+
+            /**
+             * optional string redirect = 3;
+             */
+            public Builder setRedirectBytes(com.google.protobuf.ByteString value) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                bitField0_ |= 0x00000004;
+                redirect_ = value;
+                onChanged();
+                return this;
+            }
+
+            private Object errorMsg_ = "";
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public boolean hasErrorMsg() {
+                return ((bitField0_ & 0x00000008) == 0x00000008);
+            }
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public String getErrorMsg() {
+                Object ref = errorMsg_;
+                if (!(ref instanceof String)) {
+                    com.google.protobuf.ByteString bs = (com.google.protobuf.ByteString) ref;
+                    String s = bs.toStringUtf8();
+                    if (bs.isValidUtf8()) {
+                        errorMsg_ = s;
+                    }
+                    return s;
+                } else {
+                    return (String) ref;
+                }
+            }
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public com.google.protobuf.ByteString getErrorMsgBytes() {
+                Object ref = errorMsg_;
+                if (ref instanceof String) {
+                    com.google.protobuf.ByteString b = com.google.protobuf.ByteString.copyFromUtf8((String) ref);
+                    errorMsg_ = b;
+                    return b;
+                } else {
+                    return (com.google.protobuf.ByteString) ref;
+                }
+            }
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public Builder setErrorMsg(String value) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                bitField0_ |= 0x00000008;
+                errorMsg_ = value;
+                onChanged();
+                return this;
+            }
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public Builder clearErrorMsg() {
+                bitField0_ = (bitField0_ & ~0x00000008);
+                errorMsg_ = getDefaultInstance().getErrorMsg();
+                onChanged();
+                return this;
+            }
+
+            /**
+             * optional string errorMsg = 4;
+             */
+            public Builder setErrorMsgBytes(com.google.protobuf.ByteString value) {
+                if (value == null) {
+                    throw new NullPointerException();
+                }
+                bitField0_ |= 0x00000008;
+                errorMsg_ = value;
+                onChanged();
+                return this;
+            }
+
+            public final Builder setUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.setUnknownFields(unknownFields);
+            }
+
+            public final Builder mergeUnknownFields(final com.google.protobuf.UnknownFieldSet unknownFields) {
+                return super.mergeUnknownFields(unknownFields);
+            }
+
+            // @@protoc_insertion_point(builder_scope:jraft.ValueResponse)
+        }
+
+        // @@protoc_insertion_point(class_scope:jraft.ValueResponse)
+        private static final ValueResponse DEFAULT_INSTANCE;
+        static {
+            DEFAULT_INSTANCE = new ValueResponse();
+        }
+
+        public static ValueResponse getDefaultInstance() {
+            return DEFAULT_INSTANCE;
+        }
+
+        @Deprecated
+        public static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() {
+                                                                                 public ValueResponse parsePartialFrom(com.google.protobuf.CodedInputStream input,
+                                                                                                                       com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+                                                                                                                                                                                   throws com.google.protobuf.InvalidProtocolBufferException {
+                                                                                     return new ValueResponse(input,
+                                                                                         extensionRegistry);
+                                                                                 }
+                                                                             };
+
+        public static com.google.protobuf.Parser parser() {
+            return PARSER;
+        }
+
+        @Override
+        public com.google.protobuf.Parser getParserForType() {
+            return PARSER;
+        }
+
+        public ValueResponse getDefaultInstanceForType() {
+            return DEFAULT_INSTANCE;
+        }
+
+    }
+
+    private static final com.google.protobuf.Descriptors.Descriptor                internal_static_jraft_GetValueRequest_descriptor;
+    private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_GetValueRequest_fieldAccessorTable;
+    private static final com.google.protobuf.Descriptors.Descriptor                internal_static_jraft_IncrementAndGetRequest_descriptor;
+    private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_IncrementAndGetRequest_fieldAccessorTable;
+    private static final com.google.protobuf.Descriptors.Descriptor                internal_static_jraft_ValueResponse_descriptor;
+    private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_jraft_ValueResponse_fieldAccessorTable;
+
+    public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() {
+        return descriptor;
+    }
+
+    private static com.google.protobuf.Descriptors.FileDescriptor descriptor;
+    static {
+        String[] descriptorData = { "\n\rcounter.proto\022\005jraft\"\'\n\017GetValueReques"
+                                    + "t\022\024\n\014readOnlySafe\030\001 \002(\010\"\'\n\026IncrementAndG"
+                                    + "etRequest\022\r\n\005delta\030\001 \002(\003\"S\n\rValueRespons"
+                                    + "e\022\r\n\005value\030\001 \002(\003\022\017\n\007success\030\002 \002(\010\022\020\n\010red"
+                                    + "irect\030\003 \001(\t\022\020\n\010errorMsg\030\004 \001(\tB:\n)com.ali"
+                                    + "pay.sofa.jraft.example.counter.rpcB\rCoun" + "terOutter" };
+        com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
+            public com.google.protobuf.ExtensionRegistry assignDescriptors(com.google.protobuf.Descriptors.FileDescriptor root) {
+                descriptor = root;
+                return null;
+            }
+        };
+        com.google.protobuf.Descriptors.FileDescriptor.internalBuildGeneratedFileFrom(descriptorData,
+            new com.google.protobuf.Descriptors.FileDescriptor[] {}, assigner);
+        internal_static_jraft_GetValueRequest_descriptor = getDescriptor().getMessageTypes().get(0);
+        internal_static_jraft_GetValueRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_jraft_GetValueRequest_descriptor, new String[] { "ReadOnlySafe", });
+        internal_static_jraft_IncrementAndGetRequest_descriptor = getDescriptor().getMessageTypes().get(1);
+        internal_static_jraft_IncrementAndGetRequest_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_jraft_IncrementAndGetRequest_descriptor, new String[] { "Delta", });
+        internal_static_jraft_ValueResponse_descriptor = getDescriptor().getMessageTypes().get(2);
+        internal_static_jraft_ValueResponse_fieldAccessorTable = new com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
+            internal_static_jraft_ValueResponse_descriptor,
+            new String[] { "Value", "Success", "Redirect", "ErrorMsg", });
+    }
+
+    // @@protoc_insertion_point(outer_class_scope)
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/GetValueRequestProcessor.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/GetValueRequestProcessor.java
new file mode 100644
index 0000000..3fdce5c
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/GetValueRequestProcessor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter.rpc;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.example.counter.CounterClosure;
+import com.alipay.sofa.jraft.example.counter.CounterService;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.GetValueRequest;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+
+/**
+ * GetValueRequest processor.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 5:48:33 PM
+ */
+public class GetValueRequestProcessor implements RpcProcessor {
+
+    private final CounterService counterService;
+
+    public GetValueRequestProcessor(CounterService counterService) {
+        super();
+        this.counterService = counterService;
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final GetValueRequest request) {
+        final CounterClosure closure = new CounterClosure() {
+            @Override
+            public void run(Status status) {
+                rpcCtx.sendResponse(getValueResponse());
+            }
+        };
+
+        this.counterService.get(request.getReadOnlySafe(), closure);
+    }
+
+    @Override
+    public String interest() {
+        return GetValueRequest.class.getName();
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/IncrementAndGetRequestProcessor.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/IncrementAndGetRequestProcessor.java
new file mode 100644
index 0000000..fa89f91
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/rpc/IncrementAndGetRequestProcessor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter.rpc;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.example.counter.CounterClosure;
+import com.alipay.sofa.jraft.example.counter.CounterService;
+import com.alipay.sofa.jraft.example.counter.rpc.CounterOutter.IncrementAndGetRequest;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+
+/**
+ * IncrementAndGetRequest processor.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 5:43:57 PM
+ */
+public class IncrementAndGetRequestProcessor implements RpcProcessor {
+
+    private final CounterService counterService;
+
+    public IncrementAndGetRequestProcessor(CounterService counterService) {
+        super();
+        this.counterService = counterService;
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final IncrementAndGetRequest request) {
+        final CounterClosure closure = new CounterClosure() {
+            @Override
+            public void run(Status status) {
+                rpcCtx.sendResponse(getValueResponse());
+            }
+        };
+
+        this.counterService.incrementAndGet(request.getDelta(), closure);
+    }
+
+    @Override
+    public String interest() {
+        return IncrementAndGetRequest.class.getName();
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/snapshot/CounterSnapshotFile.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/snapshot/CounterSnapshotFile.java
new file mode 100644
index 0000000..e23d0c2
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/counter/snapshot/CounterSnapshotFile.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.counter.snapshot;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Counter snapshot file.
+ *
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 5:14:55 PM
+ */
+public class CounterSnapshotFile {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CounterSnapshotFile.class);
+
+    private String              path;
+
+    public CounterSnapshotFile(String path) {
+        super();
+        this.path = path;
+    }
+
+    public String getPath() {
+        return this.path;
+    }
+
+    /**
+     * Save value to snapshot file.
+     */
+    public boolean save(final long value) {
+        try {
+            FileUtils.writeStringToFile(new File(path), String.valueOf(value));
+            return true;
+        } catch (IOException e) {
+            LOG.error("Fail to save snapshot", e);
+            return false;
+        }
+    }
+
+    public long load() throws IOException {
+        final String s = FileUtils.readFileToString(new File(path));
+        if (!StringUtils.isBlank(s)) {
+            return Long.parseLong(s);
+        }
+        throw new IOException("Fail to load snapshot from " + path + ",content: " + s);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionBootstrap.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionBootstrap.java
new file mode 100644
index 0000000..0159d2e
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionBootstrap.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.election;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ElectionBootstrap {
+
+    // Start elections by 3 instance. Note that if multiple instances are started on the same machine,
+    // the first parameter `dataPath` should not be the same.
+    public static void main(final String[] args) {
+        if (args.length < 4) {
+            System.out
+                .println("Usage : java com.alipay.sofa.jraft.example.election.ElectionBootstrap {dataPath} {groupId} {serverId} {initConf}");
+            System.out
+                .println("Example: java com.alipay.sofa.jraft.example.election.ElectionBootstrap /tmp/server1 election_test 127.0.0.1:8081 127.0.0.1:8081,127.0.0.1:8082,127.0.0.1:8083");
+            System.exit(1);
+        }
+        final String dataPath = args[0];
+        final String groupId = args[1];
+        final String serverIdStr = args[2];
+        final String initialConfStr = args[3];
+
+        final ElectionNodeOptions electionOpts = new ElectionNodeOptions();
+        electionOpts.setDataPath(dataPath);
+        electionOpts.setGroupId(groupId);
+        electionOpts.setServerAddress(serverIdStr);
+        electionOpts.setInitialServerAddressList(initialConfStr);
+
+        final ElectionNode node = new ElectionNode();
+        node.addLeaderStateListener(new LeaderStateListener() {
+
+            @Override
+            public void onLeaderStart(long leaderTerm) {
+                PeerId serverId = node.getNode().getLeaderId();
+                String ip = serverId.getIp();
+                int port = serverId.getPort();
+                System.out.println("[ElectionBootstrap] Leader's ip is: " + ip + ", port: " + port);
+                System.out.println("[ElectionBootstrap] Leader start on term: " + leaderTerm);
+            }
+
+            @Override
+            public void onLeaderStop(long leaderTerm) {
+                System.out.println("[ElectionBootstrap] Leader stop on term: " + leaderTerm);
+            }
+        });
+        node.init(electionOpts);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNode.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNode.java
new file mode 100644
index 0000000..e80ba36
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNode.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.election;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ElectionNode implements Lifecycle {
+
+    private static final Logger             LOG       = LoggerFactory.getLogger(ElectionNode.class);
+
+    private final List listeners = new CopyOnWriteArrayList<>();
+    private RaftGroupService                raftGroupService;
+    private Node                            node;
+    private ElectionOnlyStateMachine        fsm;
+
+    private boolean                         started;
+
+    @Override
+    public boolean init(final ElectionNodeOptions opts) {
+        if (this.started) {
+            LOG.info("[ElectionNode: {}] already started.", opts.getServerAddress());
+            return true;
+        }
+        // node options
+        NodeOptions nodeOpts = opts.getNodeOptions();
+        if (nodeOpts == null) {
+            nodeOpts = new NodeOptions();
+        }
+        this.fsm = new ElectionOnlyStateMachine(this.listeners);
+        nodeOpts.setFsm(this.fsm);
+        final Configuration initialConf = new Configuration();
+        if (!initialConf.parse(opts.getInitialServerAddressList())) {
+            throw new IllegalArgumentException("Fail to parse initConf: " + opts.getInitialServerAddressList());
+        }
+        // Set the initial cluster configuration
+        nodeOpts.setInitialConf(initialConf);
+        final String dataPath = opts.getDataPath();
+        try {
+            FileUtils.forceMkdir(new File(dataPath));
+        } catch (final IOException e) {
+            LOG.error("Fail to make dir for dataPath {}.", dataPath);
+            return false;
+        }
+        // Set the data path
+        // Log, required
+        nodeOpts.setLogUri(Paths.get(dataPath, "log").toString());
+        // Metadata, required
+        nodeOpts.setRaftMetaUri(Paths.get(dataPath, "meta").toString());
+        // nodeOpts.setSnapshotUri(Paths.get(dataPath, "snapshot").toString());
+
+        final String groupId = opts.getGroupId();
+        final PeerId serverId = new PeerId();
+        if (!serverId.parse(opts.getServerAddress())) {
+            throw new IllegalArgumentException("Fail to parse serverId: " + opts.getServerAddress());
+        }
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint());
+        this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOpts, rpcServer);
+        this.node = this.raftGroupService.start();
+        if (this.node != null) {
+            this.started = true;
+        }
+        return this.started;
+    }
+
+    @Override
+    public void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        if (this.raftGroupService != null) {
+            this.raftGroupService.shutdown();
+            try {
+                this.raftGroupService.join();
+            } catch (final InterruptedException e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        this.started = false;
+        LOG.info("[ElectionNode] shutdown successfully: {}.", this);
+    }
+
+    public Node getNode() {
+        return node;
+    }
+
+    public ElectionOnlyStateMachine getFsm() {
+        return fsm;
+    }
+
+    public boolean isStarted() {
+        return started;
+    }
+
+    public boolean isLeader() {
+        return this.fsm.isLeader();
+    }
+
+    public void addLeaderStateListener(final LeaderStateListener listener) {
+        this.listeners.add(listener);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNodeOptions.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNodeOptions.java
new file mode 100644
index 0000000..56d8be7
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionNodeOptions.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.election;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ElectionNodeOptions {
+
+    private String      dataPath;
+    // raft group id
+    private String      groupId;
+    // ip:port
+    private String      serverAddress;
+    // ip:port,ip:port,ip:port
+    private String      initialServerAddressList;
+    // raft node options
+    private NodeOptions nodeOptions;
+
+    public String getDataPath() {
+        return dataPath;
+    }
+
+    public void setDataPath(String dataPath) {
+        this.dataPath = dataPath;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getServerAddress() {
+        return serverAddress;
+    }
+
+    public void setServerAddress(String serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    public String getInitialServerAddressList() {
+        return initialServerAddressList;
+    }
+
+    public void setInitialServerAddressList(String initialServerAddressList) {
+        this.initialServerAddressList = initialServerAddressList;
+    }
+
+    public NodeOptions getNodeOptions() {
+        return nodeOptions;
+    }
+
+    public void setNodeOptions(NodeOptions nodeOptions) {
+        this.nodeOptions = nodeOptions;
+    }
+
+    @Override
+    public String toString() {
+        return "ElectionNodeOptions{" + "dataPath='" + dataPath + '\'' + ", groupId='" + groupId + '\''
+               + ", serverAddress='" + serverAddress + '\'' + ", initialServerAddressList='" + initialServerAddressList
+               + '\'' + ", nodeOptions=" + nodeOptions + '}';
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionOnlyStateMachine.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionOnlyStateMachine.java
new file mode 100644
index 0000000..6db6e3a
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/ElectionOnlyStateMachine.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.election;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ElectionOnlyStateMachine extends StateMachineAdapter {
+
+    private static final Logger             LOG        = LoggerFactory.getLogger(ElectionOnlyStateMachine.class);
+
+    private final AtomicLong                leaderTerm = new AtomicLong(-1L);
+    private final List listeners;
+
+    public ElectionOnlyStateMachine(List listeners) {
+        this.listeners = listeners;
+    }
+
+    @Override
+    public void onApply(final Iterator it) {
+        // election only, do nothing
+        while (it.hasNext()) {
+            LOG.info("On apply with term: {} and index: {}. ", it.getTerm(), it.getIndex());
+            it.next();
+        }
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        super.onLeaderStart(term);
+        this.leaderTerm.set(term);
+        for (final LeaderStateListener listener : this.listeners) { // iterator the snapshot
+            listener.onLeaderStart(term);
+        }
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        super.onLeaderStop(status);
+        final long oldTerm = leaderTerm.get();
+        this.leaderTerm.set(-1L);
+        for (final LeaderStateListener listener : this.listeners) { // iterator the snapshot
+            listener.onLeaderStop(oldTerm);
+        }
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    public void addLeaderStateListener(final LeaderStateListener listener) {
+        this.listeners.add(listener);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/LeaderStateListener.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/LeaderStateListener.java
new file mode 100644
index 0000000..d3564cf
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/election/LeaderStateListener.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.election;
+
+/**
+ * Leader state listener.
+ *
+ * @author dennis
+ */
+public interface LeaderStateListener {
+
+    /**
+     * Called when current node becomes leader
+     */
+    void onLeaderStart(final long leaderTerm);
+
+    /**
+     * Called when current node loses leadership.
+     */
+    void onLeaderStop(final long leaderTerm);
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/LeaderStateListener.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/LeaderStateListener.java
new file mode 100644
index 0000000..0fc2b17
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/LeaderStateListener.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.priorityelection;
+
+/**
+ * @author zongtanghu
+ */
+public interface LeaderStateListener {
+
+    /**
+     * Called when current node becomes leader
+     */
+    void onLeaderStart(final long leaderTerm);
+
+    /**
+     * Called when current node loses leadership.
+     */
+    void onLeaderStop(final long leaderTerm);
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionBootstrap.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionBootstrap.java
new file mode 100644
index 0000000..5ded409
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionBootstrap.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.priorityelection;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ *
+ * @author zongtanghu
+ */
+public class PriorityElectionBootstrap {
+
+    // Start elections by 3 instance. Note that if multiple instances are started on the same machine,
+    // the first parameter `dataPath` should not be the same,
+    // the second parameter `groupId` should be set the same value, eg: election_test,
+    // the third parameter `serverId` should be set ip and port with priority value, eg: 127.0.0.1:8081::100, and middle postion can be empty string,
+    // the fourth parameter `initialConfStr` should be set the all of endpoints in raft cluster, eg : 127.0.0.1:8081::100,127.0.0.1:8082::40,127.0.0.1:8083::40.
+
+    public static void main(final String[] args) {
+        if (args.length < 4) {
+            System.out
+                .println("Usage : java com.alipay.sofa.jraft.example.priorityelection.PriorityElectionBootstrap {dataPath} {groupId} {serverId} {initConf}");
+            System.out
+                .println("Example: java com.alipay.sofa.jraft.example.priorityelection.PriorityElectionBootstrap /tmp/server1 election_test 127.0.0.1:8081::100 127.0.0.1:8081::100,127.0.0.1:8082::40,127.0.0.1:8083::40");
+            System.exit(1);
+        }
+        final String dataPath = args[0];
+        final String groupId = args[1];
+        final String serverIdStr = args[2];
+        final String initialConfStr = args[3];
+
+        final PriorityElectionNodeOptions priorityElectionOpts = new PriorityElectionNodeOptions();
+        priorityElectionOpts.setDataPath(dataPath);
+        priorityElectionOpts.setGroupId(groupId);
+        priorityElectionOpts.setServerAddress(serverIdStr);
+        priorityElectionOpts.setInitialServerAddressList(initialConfStr);
+
+        final PriorityElectionNode node = new PriorityElectionNode();
+        node.addLeaderStateListener(new LeaderStateListener() {
+
+            @Override
+            public void onLeaderStart(long leaderTerm) {
+
+                PeerId serverId = node.getNode().getLeaderId();
+                int priority = serverId.getPriority();
+                String ip = serverId.getIp();
+                int port = serverId.getPort();
+
+                System.out.println("[PriorityElectionBootstrap] Leader's ip is: " + ip + ", port: " + port
+                                   + ", priority: " + priority);
+                System.out.println("[PriorityElectionBootstrap] Leader start on term: " + leaderTerm);
+            }
+
+            @Override
+            public void onLeaderStop(long leaderTerm) {
+                System.out.println("[PriorityElectionBootstrap] Leader stop on term: " + leaderTerm);
+            }
+        });
+        node.init(priorityElectionOpts);
+    }
+
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNode.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNode.java
new file mode 100644
index 0000000..86f2c8d
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNode.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.priorityelection;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+/**
+ * @author zongtanghu
+ */
+public class PriorityElectionNode implements Lifecycle {
+
+    private static final Logger              LOG       = LoggerFactory.getLogger(PriorityElectionNode.class);
+
+    private final List  listeners = new CopyOnWriteArrayList<>();
+    private RaftGroupService                 raftGroupService;
+    private Node                             node;
+    private PriorityElectionOnlyStateMachine fsm;
+
+    private boolean                          started;
+
+    @Override
+    public boolean init(final PriorityElectionNodeOptions opts) {
+        if (this.started) {
+            LOG.info("[PriorityElectionNode: {}] already started.", opts.getServerAddress());
+            return true;
+        }
+        // node options
+        NodeOptions nodeOpts = opts.getNodeOptions();
+        if (nodeOpts == null) {
+            nodeOpts = new NodeOptions();
+        }
+        this.fsm = new PriorityElectionOnlyStateMachine(this.listeners);
+        // Set the initial PriorityElectionOnlyStateMachine
+        nodeOpts.setFsm(this.fsm);
+        final Configuration initialConf = new Configuration();
+        if (!initialConf.parse(opts.getInitialServerAddressList())) {
+            throw new IllegalArgumentException("Fail to parse initConf: " + opts.getInitialServerAddressList());
+        }
+        // Set the initial cluster configuration
+        nodeOpts.setInitialConf(initialConf);
+        final String dataPath = opts.getDataPath();
+        try {
+            FileUtils.forceMkdir(new File(dataPath));
+        } catch (final IOException e) {
+            LOG.error("Fail to make dir for dataPath {}.", dataPath);
+            return false;
+        }
+        // Set the data path
+        // Log, required
+        nodeOpts.setLogUri(Paths.get(dataPath, "log").toString());
+        // Metadata, required
+        nodeOpts.setRaftMetaUri(Paths.get(dataPath, "meta").toString());
+        // nodeOpts.setSnapshotUri(Paths.get(dataPath, "snapshot").toString());
+
+        final String groupId = opts.getGroupId();
+        final PeerId serverId = new PeerId();
+        if (!serverId.parse(opts.getServerAddress())) {
+            throw new IllegalArgumentException("Fail to parse serverId: " + opts.getServerAddress());
+        }
+
+        /**
+         * Set priority value, required for priority-based election, it must be a positive value when
+         * enable the feature, some special value meaning:
+         * 

+         * value.
+         */
+        nodeOpts.setElectionPriority(serverId.getPriority());
+
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint());
+        this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOpts, rpcServer);
+        this.node = this.raftGroupService.start();
+        if (this.node != null) {
+            this.started = true;
+        }
+        return this.started;
+    }
+
+    @Override
+    public void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        if (this.raftGroupService != null) {
+            this.raftGroupService.shutdown();
+            try {
+                this.raftGroupService.join();
+            } catch (final InterruptedException e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        this.started = false;
+        LOG.info("[ElectionNode] shutdown successfully: {}.", this);
+    }
+
+    public Node getNode() {
+        return this.node;
+    }
+
+    public PriorityElectionOnlyStateMachine getFsm() {
+        return this.fsm;
+    }
+
+    public boolean isStarted() {
+        return this.started;
+    }
+
+    public boolean isLeader() {
+        return this.fsm.isLeader();
+    }
+
+    public void addLeaderStateListener(final LeaderStateListener listener) {
+        this.listeners.add(listener);
+    }
+
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNodeOptions.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNodeOptions.java
new file mode 100644
index 0000000..9f56309
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionNodeOptions.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.priorityelection;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+
+/**
+ * @author zongtanghu
+ */
+public class PriorityElectionNodeOptions {
+
+    private String      dataPath;
+    // raft group id
+    private String      groupId;
+    // ip:port::priority
+    private String      serverAddress;
+    // ip:port::priority,ip:port::priority,ip:port::priority
+    private String      initialServerAddressList;
+    // raft node options
+    private NodeOptions nodeOptions;
+
+    public String getDataPath() {
+        return dataPath;
+    }
+
+    public void setDataPath(String dataPath) {
+        this.dataPath = dataPath;
+    }
+
+    public String getGroupId() {
+        return groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getServerAddress() {
+        return serverAddress;
+    }
+
+    public void setServerAddress(String serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    public String getInitialServerAddressList() {
+        return initialServerAddressList;
+    }
+
+    public void setInitialServerAddressList(String initialServerAddressList) {
+        this.initialServerAddressList = initialServerAddressList;
+    }
+
+    public NodeOptions getNodeOptions() {
+        return nodeOptions;
+    }
+
+    public void setNodeOptions(NodeOptions nodeOptions) {
+        this.nodeOptions = nodeOptions;
+    }
+
+    @Override
+    public String toString() {
+        return "PriorityElectionNodeOptions{" + "dataPath='" + dataPath + '\'' + ", groupId='" + groupId + '\''
+               + ", serverAddress='" + serverAddress + '\'' + ", initialServerAddressList='" + initialServerAddressList
+               + '\'' + ", nodeOptions=" + nodeOptions + '}';
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionOnlyStateMachine.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionOnlyStateMachine.java
new file mode 100644
index 0000000..d8c200c
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/priorityelection/PriorityElectionOnlyStateMachine.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.priorityelection;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+
+/**
+ * @author zongtanghu
+ */
+public class PriorityElectionOnlyStateMachine extends StateMachineAdapter {
+
+    private static final Logger             LOG        = LoggerFactory
+                                                           .getLogger(PriorityElectionOnlyStateMachine.class);
+
+    private final AtomicLong                leaderTerm = new AtomicLong(-1L);
+    private final List listeners;
+
+    public PriorityElectionOnlyStateMachine(List listeners) {
+        this.listeners = listeners;
+    }
+
+    @Override
+    public void onApply(final Iterator it) {
+        // election only, do nothing
+        while (it.hasNext()) {
+            LOG.info("On apply with term: {} and index: {}. ", it.getTerm(), it.getIndex());
+            it.next();
+        }
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        super.onLeaderStart(term);
+        this.leaderTerm.set(term);
+        for (final LeaderStateListener listener : this.listeners) { // iterator the snapshot
+            listener.onLeaderStart(term);
+        }
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        super.onLeaderStop(status);
+        final long oldTerm = leaderTerm.get();
+        this.leaderTerm.set(-1L);
+        for (final LeaderStateListener listener : this.listeners) { // iterator the snapshot
+            listener.onLeaderStop(oldTerm);
+        }
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    public void addLeaderStateListener(final LeaderStateListener listener) {
+        this.listeners.add(listener);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Client.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Client.java
new file mode 100644
index 0000000..6f42acf
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Client.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionRouteTableOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.MultiRegionRouteTableOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.PlacementDriverOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RheaKVStoreOptionsConfigured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Client {
+
+    private final RheaKVStore rheaKVStore = new DefaultRheaKVStore();
+
+    public void init() {
+        final List regionRouteTableOptionsList = MultiRegionRouteTableOptionsConfigured
+            .newConfigured() //
+            .withInitialServerList(-1L /* default id */, Configs.ALL_NODE_ADDRESSES) //
+            .config();
+        final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured() //
+            .withFake(true) //
+            .withRegionRouteTableOptionsList(regionRouteTableOptionsList) //
+            .config();
+        final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+            .withClusterName(Configs.CLUSTER_NAME) //
+            .withPlacementDriverOptions(pdOpts) //
+            .config();
+        System.out.println(opts);
+        rheaKVStore.init(opts);
+    }
+
+    public void shutdown() {
+        this.rheaKVStore.shutdown();
+    }
+
+    public RheaKVStore getRheaKVStore() {
+        return rheaKVStore;
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/CompareAndPutExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/CompareAndPutExample.java
new file mode 100644
index 0000000..de0951e
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/CompareAndPutExample.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CompletableFuture;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class CompareAndPutExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(CompareAndPutExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        put(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    private static void put(final RheaKVStore rheaKVStore) {
+        final CompletableFuture r1 = rheaKVStore.put("compareAndPut", writeUtf8("compareAndPutExpect"));
+        if (FutureHelper.get(r1)) {
+            LOG.info("Async put compareAndPut {} success.", readUtf8(rheaKVStore.bGet("compareAndPut")));
+        }
+
+        final CompletableFuture f1 = rheaKVStore.compareAndPut(writeUtf8("compareAndPut"),
+            writeUtf8("compareAndPutExpect"), writeUtf8("compareAndPutUpdate"));
+        if (FutureHelper.get(f1)) {
+            LOG.info("Compare compareAndPutExpect and set {} success.", readUtf8(rheaKVStore.bGet("compareAndPut")));
+        }
+
+        final CompletableFuture f2 = rheaKVStore.compareAndPut("compareAndPut",
+            writeUtf8("compareAndPutUpdate"), writeUtf8("compareAndPutUpdate2"));
+        if (FutureHelper.get(f2)) {
+            LOG.info("Compare compareAndPutUpdate and set {} success.", readUtf8(rheaKVStore.bGet("compareAndPut")));
+        }
+
+        final Boolean b1 = rheaKVStore.bCompareAndPut(writeUtf8("compareAndPut1"), writeUtf8("compareAndPutUpdate2"),
+            writeUtf8("compareAndPutUpdate3"));
+        if (b1) {
+            LOG.info("Compare compareAndPutUpdate2 and set {} success.", readUtf8(rheaKVStore.bGet("compareAndPut")));
+        }
+
+        final Boolean b2 = rheaKVStore.bCompareAndPut(writeUtf8("compareAndPut1"), writeUtf8("compareAndPutUpdate3"),
+            writeUtf8("compareAndPutUpdate4"));
+        if (b2) {
+            LOG.info("Compare compareAndPutUpdate3 and set {} success.", readUtf8(rheaKVStore.bGet("compareAndPut")));
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Configs.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Configs.java
new file mode 100644
index 0000000..71978bd
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Configs.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.io.File;
+
+/**
+ * @author jiachun.fjc
+ */
+public class Configs {
+
+    public static String DB_PATH            = "rhea_db" + File.separator;
+
+    public static String RAFT_DATA_PATH     = "raft_data" + File.separator;
+
+    public static String ALL_NODE_ADDRESSES = "127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183";
+
+    public static String CLUSTER_NAME       = "rhea_example";
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteExample.java
new file mode 100644
index 0000000..87d7693
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteExample.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DeleteExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        delete(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void delete(final RheaKVStore rheaKVStore) {
+        rheaKVStore.bPut("delete_test", writeUtf8("1"));
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+        final CompletableFuture f1 = rheaKVStore.delete(writeUtf8("delete_test"));
+        FutureHelper.get(f1);
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+
+        rheaKVStore.bPut("delete_test", writeUtf8("1"));
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+        final CompletableFuture f2 = rheaKVStore.delete("delete_test");
+        FutureHelper.get(f2);
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+
+        rheaKVStore.bPut("delete_test", writeUtf8("1"));
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+        rheaKVStore.bDelete(writeUtf8("delete_test"));
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+
+        rheaKVStore.bPut("delete_test", writeUtf8("1"));
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+        rheaKVStore.bDelete("delete_test");
+        LOG.info("Value={}", readUtf8(rheaKVStore.bGet("delete_test")));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteRangeExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteRangeExample.java
new file mode 100644
index 0000000..9d56fe0
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DeleteRangeExample.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteRangeExample {
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        deleteRange(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void deleteRange(final RheaKVStore rheaKVStore) {
+        for (int i = 0; i < 10; i++) {
+            rheaKVStore.bPut("delete_range_example_" + i, writeUtf8("1"));
+        }
+        final byte[] start = writeUtf8("delete_range_example_0");
+        final byte[] end = writeUtf8("delete_range_example_9");
+        final CompletableFuture f1 = rheaKVStore.deleteRange(start, end);
+        FutureHelper.get(f1);
+
+        for (int i = 0; i < 10; i++) {
+            rheaKVStore.bPut("delete_range_example_" + i, writeUtf8("1"));
+        }
+        final CompletableFuture f2 = rheaKVStore.deleteRange("delete_range_example_0",
+            "delete_range_example_9");
+        FutureHelper.get(f2);
+
+        for (int i = 0; i < 10; i++) {
+            rheaKVStore.bPut("delete_range_example_" + i, writeUtf8("1"));
+        }
+        rheaKVStore.bDeleteRange(start, end);
+
+        for (int i = 0; i < 10; i++) {
+            rheaKVStore.bPut("delete_range_example_" + i, writeUtf8("1"));
+        }
+        rheaKVStore.bDeleteRange("delete_range_example_0", "delete_range_example_9");
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DistributedLockExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DistributedLockExample.java
new file mode 100644
index 0000000..9b7983b
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/DistributedLockExample.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DistributedLockExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DistributedLockExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        lock(client.getRheaKVStore());
+        lockAndAutoKeepLease(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void lock(final RheaKVStore rheaKVStore) {
+        final String lockKey = "lock_example";
+        final DistributedLock lock = rheaKVStore.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        if (lock.tryLock()) {
+            try {
+                LOG.info("Lock success with: {}", lockKey);
+            } finally {
+                lock.unlock();
+            }
+        } else {
+            LOG.info("Fail to lock with: {}", lockKey);
+        }
+    }
+
+    public static void lockAndAutoKeepLease(final RheaKVStore rheaKVStore) {
+        final ScheduledExecutorService watchdog = Executors.newSingleThreadScheduledExecutor();
+        final String lockKey = "lock_example1";
+        final DistributedLock lock = rheaKVStore.getDistributedLock(lockKey, 3, TimeUnit.SECONDS, watchdog);
+        if (lock.tryLock()) {
+            try {
+                LOG.info("Lock success with: {}", lockKey);
+            } finally {
+                lock.unlock();
+            }
+        } else {
+            LOG.info("Fail to lock with: {}", lockKey);
+        }
+        ExecutorServiceHelper.shutdownAndAwaitTermination(watchdog);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetAndPutExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetAndPutExample.java
new file mode 100644
index 0000000..0560e83
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetAndPutExample.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetAndPutExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GetAndPutExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        put(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void put(final RheaKVStore rheaKVStore) {
+        final CompletableFuture f1 = rheaKVStore.getAndPut(writeUtf8("getAndPut"), writeUtf8("getAndPutValue"));
+        LOG.info("Old value: {}", readUtf8(FutureHelper.get(f1)));
+
+        final CompletableFuture f2 = rheaKVStore.getAndPut("getAndPut", writeUtf8("getAndPutValue2"));
+        LOG.info("Old value: {}", readUtf8(FutureHelper.get(f2)));
+
+        final byte[] b1 = rheaKVStore.bGetAndPut(writeUtf8("getAndPut1"), writeUtf8("getAndPutValue3"));
+        LOG.info("Old value: {}", readUtf8(b1));
+
+        final byte[] b2 = rheaKVStore.bGetAndPut(writeUtf8("getAndPut1"), writeUtf8("getAndPutValue4"));
+        LOG.info("Old value: {}", readUtf8(b2));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetExample.java
new file mode 100644
index 0000000..f522777
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetExample.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GetExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        get(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void get(final RheaKVStore rheaKVStore) {
+        final byte[] key = writeUtf8("hello");
+        final byte[] value = writeUtf8("world");
+        rheaKVStore.bPut(key, value);
+
+        // async get with bytes
+        final CompletableFuture f1 = rheaKVStore.get(key);
+        final CompletableFuture f2 = rheaKVStore.get(key, false);
+        // async get with string
+        final CompletableFuture f3 = rheaKVStore.get("hello");
+        final CompletableFuture f4 = rheaKVStore.get("hello", false);
+        CompletableFuture.allOf(f1, f2, f3, f4).join();
+        LOG.info("Async get result={}", readUtf8(f1.join()));
+        LOG.info("Async get result={}", readUtf8(f2.join()));
+        LOG.info("Async get result={}", readUtf8(f3.join()));
+        LOG.info("Async get result={}", readUtf8(f4.join()));
+
+        // sync get with bytes
+        final byte[] b1 = rheaKVStore.bGet(key);
+        final byte[] b2 = rheaKVStore.bGet(key, false);
+        // sync get with string
+        final byte[] b3 = rheaKVStore.bGet("hello");
+        final byte[] b4 = rheaKVStore.bGet("hello", false);
+        LOG.info("Sync get result={}", readUtf8(b1));
+        LOG.info("Sync get result={}", readUtf8(b2));
+        LOG.info("Sync get result={}", readUtf8(b3));
+        LOG.info("Sync get result={}", readUtf8(b4));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetSequenceExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetSequenceExample.java
new file mode 100644
index 0000000..5a0a64d
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/GetSequenceExample.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetSequenceExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GetSequenceExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        getSequence(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void getSequence(final RheaKVStore rheaKVStore) {
+        final byte[] key = writeUtf8("sequence");
+        rheaKVStore.getSequence(key, 10);
+
+        // async
+        final CompletableFuture f1 = rheaKVStore.getSequence(key, 20);
+        final CompletableFuture f2 = rheaKVStore.getSequence("sequence", 30);
+        CompletableFuture.allOf(f1, f2).join();
+        LOG.info("Async getSequence result={}", f1.join());
+        LOG.info("Async getSequence result={}", f2.join());
+
+        final CompletableFuture f3 = rheaKVStore.resetSequence(key);
+        f3.join();
+
+        // sync
+        final Sequence b1 = rheaKVStore.bGetSequence(key, 40);
+        final Sequence b2 = rheaKVStore.bGetSequence("sequence", 50);
+        LOG.info("Sync getSequence result={}", b1);
+        LOG.info("Sync getSequence result={}", b2);
+
+        rheaKVStore.bResetSequence(key);
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/IteratorExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/IteratorExample.java
new file mode 100644
index 0000000..ab60bb7
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/IteratorExample.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaIterator;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class IteratorExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(IteratorExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        iterator(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    @SuppressWarnings("unchecked")
+    public static void iterator(final RheaKVStore rheaKVStore) {
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] bytes = writeUtf8("iterator_demo_" + i);
+            keys.add(bytes);
+            rheaKVStore.bPut(bytes, bytes);
+        }
+
+        final byte[] firstKey = keys.get(0);
+        final byte[] lastKey = keys.get(keys.size() - 1);
+        final String firstKeyString = readUtf8(firstKey);
+        final String lastKeyString = readUtf8(lastKey);
+
+        final RheaIterator it1 = rheaKVStore.iterator(firstKey, lastKey, 5);
+        final RheaIterator it2 = rheaKVStore.iterator(firstKey, lastKey, 6, false);
+        final RheaIterator it3 = rheaKVStore.iterator(firstKeyString, lastKeyString, 5);
+        final RheaIterator it4 = rheaKVStore.iterator(firstKeyString, lastKeyString, 6, false);
+
+        for (final RheaIterator it : new RheaIterator[] { it1, it2, it3, it4 }) {
+            while (it.hasNext()) {
+                final KVEntry kv = it.next();
+                LOG.info("Sync iterator: key={}, value={}", readUtf8(kv.getKey()), readUtf8(kv.getValue()));
+            }
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MergeExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MergeExample.java
new file mode 100644
index 0000000..f0b1f3c
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MergeExample.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MergeExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MergeExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        merge(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void merge(final RheaKVStore rheaKVStore) {
+        final CompletableFuture f1 = rheaKVStore.merge("merge_example", "1");
+        final CompletableFuture f2 = rheaKVStore.merge("merge_example", "2");
+        final CompletableFuture f3 = rheaKVStore.merge("merge_example", "3");
+        final CompletableFuture f4 = rheaKVStore.merge("merge_example", "4");
+        final CompletableFuture f5 = rheaKVStore.merge("merge_example", "5");
+
+        CompletableFuture.allOf(f1, f2, f3, f4, f5).join();
+        LOG.info("Merge result is: {}", readUtf8(rheaKVStore.bGet("merge_example")));
+
+        rheaKVStore.bMerge("merge_example1", "1");
+        rheaKVStore.bMerge("merge_example1", "2");
+        rheaKVStore.bMerge("merge_example1", "3");
+        rheaKVStore.bMerge("merge_example1", "4");
+        rheaKVStore.bMerge("merge_example1", "5");
+        LOG.info("Merge result is: {}", readUtf8(rheaKVStore.bGet("merge_example1")));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MultiGetExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MultiGetExample.java
new file mode 100644
index 0000000..a065732
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/MultiGetExample.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MultiGetExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(MultiGetExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        multiGet(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void multiGet(final RheaKVStore rheaKVStore) {
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] bytes = writeUtf8("multi_get_demo_" + i);
+            keys.add(bytes);
+            rheaKVStore.bPut(bytes, bytes);
+        }
+
+        // async multiGet
+        final CompletableFuture> f1 = rheaKVStore.multiGet(keys);
+        final CompletableFuture> f2 = rheaKVStore.multiGet(keys, false);
+        CompletableFuture.allOf(f1, f2).join();
+        for (Map.Entry entry : f1.join().entrySet()) {
+            LOG.info("Async multiGet: key={}, value={}", readUtf8(entry.getKey().getBytes()),
+                readUtf8(entry.getValue()));
+        }
+        for (Map.Entry entry : f2.join().entrySet()) {
+            LOG.info("Async multiGet: key={}, value={}", readUtf8(entry.getKey().getBytes()),
+                readUtf8(entry.getValue()));
+        }
+
+        // async multiGet
+        final Map map1 = rheaKVStore.bMultiGet(keys);
+        Map map2 = rheaKVStore.bMultiGet(keys, false);
+        for (Map.Entry entry : map1.entrySet()) {
+            LOG.info("Sync multiGet: key={}, value={}", readUtf8(entry.getKey().getBytes()), readUtf8(entry.getValue()));
+        }
+        for (Map.Entry entry : map2.entrySet()) {
+            LOG.info("Sync multiGet: key={}, value={}", readUtf8(entry.getKey().getBytes()), readUtf8(entry.getValue()));
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Node.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Node.java
new file mode 100644
index 0000000..7ef10a8
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Node.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Node {
+
+    private final RheaKVStoreOptions options;
+
+    private RheaKVStore              rheaKVStore;
+
+    public Node(RheaKVStoreOptions options) {
+        this.options = options;
+    }
+
+    public void start() {
+        this.rheaKVStore = new DefaultRheaKVStore();
+        this.rheaKVStore.init(this.options);
+    }
+
+    public void stop() {
+        this.rheaKVStore.shutdown();
+    }
+
+    public RheaKVStore getRheaKVStore() {
+        return rheaKVStore;
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutExample.java
new file mode 100644
index 0000000..e8de1ba
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutExample.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PutExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        put(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void put(final RheaKVStore rheaKVStore) {
+        final byte[] value = writeUtf8("put_example_value");
+        final CompletableFuture r1 = rheaKVStore.put("1", value);
+        if (FutureHelper.get(r1)) {
+            LOG.info("Async put 1 {} success.", readUtf8(rheaKVStore.bGet("1")));
+        }
+
+        final CompletableFuture r2 = rheaKVStore.put(writeUtf8("2"), value);
+        if (FutureHelper.get(r2)) {
+            LOG.info("Async put 2 {} success.", readUtf8(rheaKVStore.bGet("2")));
+        }
+
+        final boolean r3 = rheaKVStore.bPut("3", value);
+        if (r3) {
+            LOG.info("Sync put 3 {} success.", readUtf8(rheaKVStore.bGet("3")));
+        }
+
+        final boolean r4 = rheaKVStore.bPut(writeUtf8("4"), value);
+        if (r4) {
+            LOG.info("Sync put 4 {} success.", readUtf8(rheaKVStore.bGet("4")));
+        }
+
+        // put list
+        final KVEntry kv1 = new KVEntry(writeUtf8("10"), value);
+        final KVEntry kv2 = new KVEntry(writeUtf8("11"), value);
+        final KVEntry kv3 = new KVEntry(writeUtf8("12"), value);
+        final KVEntry kv4 = new KVEntry(writeUtf8("13"), value);
+        final KVEntry kv5 = new KVEntry(writeUtf8("14"), value);
+
+        List entries = Lists.newArrayList(kv1, kv2, kv3);
+
+        final CompletableFuture r5 = rheaKVStore.put(entries);
+        if (FutureHelper.get(r5)) {
+            for (final KVEntry entry : entries) {
+                LOG.info("Async put list {} with value {} success.", readUtf8(entry.getKey()),
+                    readUtf8(entry.getValue()));
+            }
+        }
+
+        entries = Lists.newArrayList(kv3, kv4, kv5);
+        final boolean r6 = rheaKVStore.bPut(entries);
+        if (r6) {
+            for (final KVEntry entry : entries) {
+                LOG.info("Sync put list {} with value {} success.", readUtf8(entry.getKey()),
+                    readUtf8(entry.getValue()));
+            }
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutIfAbsentExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutIfAbsentExample.java
new file mode 100644
index 0000000..e3ea17f
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/PutIfAbsentExample.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutIfAbsentExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PutIfAbsentExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        putIfAbsent(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    public static void putIfAbsent(final RheaKVStore rheaKVStore) {
+        final CompletableFuture r1 = rheaKVStore.putIfAbsent(writeUtf8("putIfAbsent1"), writeUtf8("1"));
+        LOG.info("Async putIfAbsent, prev value={}", readUtf8(FutureHelper.get(r1)));
+        final CompletableFuture r2 = rheaKVStore.putIfAbsent("putIfAbsent1", writeUtf8("2"));
+        LOG.info("Async putIfAbsent, prev value={}", readUtf8(FutureHelper.get(r2)));
+
+        final byte[] b1 = rheaKVStore.bPutIfAbsent(writeUtf8("putIfAbsent2"), writeUtf8("3"));
+        LOG.info("Sync putIfAbsent, prev value={}", readUtf8(b1));
+        final byte[] b2 = rheaKVStore.bPutIfAbsent(writeUtf8("putIfAbsent2"), writeUtf8("4"));
+        LOG.info("Sync putIfAbsent, prev value={}", readUtf8(b2));
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ReverseScanExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ReverseScanExample.java
new file mode 100644
index 0000000..6f8e0a4
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ReverseScanExample.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author baozi
+ */
+public class ReverseScanExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ReverseScanExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        scan(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    @SuppressWarnings("unchecked")
+    public static void scan(final RheaKVStore rheaKVStore) {
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] bytes = writeUtf8("scan_demo_" + i);
+            keys.add(bytes);
+            rheaKVStore.bPut(bytes, bytes);
+        }
+
+        final byte[] firstKey = keys.get(keys.size() - 1);
+        final byte[] lastKey = keys.get(0);
+        final String firstKeyString = readUtf8(firstKey);
+        final String lastKeyString = readUtf8(lastKey);
+
+        // async scan
+        final CompletableFuture> f1 = rheaKVStore.reverseScan(firstKey, lastKey);
+        final CompletableFuture> f2 = rheaKVStore.reverseScan(firstKey, lastKey, false);
+        final CompletableFuture> f3 = rheaKVStore.reverseScan(firstKeyString, lastKeyString);
+        final CompletableFuture> f4 = rheaKVStore.reverseScan(firstKeyString, lastKeyString, false);
+        CompletableFuture.allOf(f1, f2, f3, f4).join();
+        for (final CompletableFuture> f : new CompletableFuture[] { f1, f2, f3, f4 }) {
+            for (final KVEntry kv : f.join()) {
+                LOG.info("Async reverseScan: key={}, value={}", readUtf8(kv.getKey()), readUtf8(kv.getValue()));
+            }
+        }
+
+        // sync scan
+        final List l1 = rheaKVStore.bReverseScan(firstKey, lastKey);
+        final List l2 = rheaKVStore.bReverseScan(firstKey, lastKey, false);
+        final List l3 = rheaKVStore.bReverseScan(firstKeyString, lastKeyString);
+        final List l4 = rheaKVStore.bReverseScan(firstKeyString, lastKeyString, false);
+        for (final List l : new List[] { l1, l2, l3, l4 }) {
+            for (final KVEntry kv : l) {
+                LOG.info("sync reverseScan: key={}, value={}", readUtf8(kv.getKey()), readUtf8(kv.getValue()));
+            }
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ScanExample.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ScanExample.java
new file mode 100644
index 0000000..a169da9
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/ScanExample.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+import static com.alipay.sofa.jraft.util.BytesUtil.readUtf8;
+import static com.alipay.sofa.jraft.util.BytesUtil.writeUtf8;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ScanExample {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ScanExample.class);
+
+    public static void main(final String[] args) throws Exception {
+        final Client client = new Client();
+        client.init();
+        scan(client.getRheaKVStore());
+        client.shutdown();
+    }
+
+    @SuppressWarnings("unchecked")
+    public static void scan(final RheaKVStore rheaKVStore) {
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] bytes = writeUtf8("scan_demo_" + i);
+            keys.add(bytes);
+            rheaKVStore.bPut(bytes, bytes);
+        }
+
+        final byte[] firstKey = keys.get(0);
+        final byte[] lastKey = keys.get(keys.size() - 1);
+        final String firstKeyString = readUtf8(firstKey);
+        final String lastKeyString = readUtf8(lastKey);
+
+        // async scan
+        final CompletableFuture> f1 = rheaKVStore.scan(firstKey, lastKey);
+        final CompletableFuture> f2 = rheaKVStore.scan(firstKey, lastKey, false);
+        final CompletableFuture> f3 = rheaKVStore.scan(firstKeyString, lastKeyString);
+        final CompletableFuture> f4 = rheaKVStore.scan(firstKeyString, lastKeyString, false);
+        CompletableFuture.allOf(f1, f2, f3, f4).join();
+        for (final CompletableFuture> f : new CompletableFuture[] { f1, f2, f3, f4 }) {
+            for (final KVEntry kv : f.join()) {
+                LOG.info("Async scan: key={}, value={}", readUtf8(kv.getKey()), readUtf8(kv.getValue()));
+            }
+        }
+
+        // sync scan
+        final List l1 = rheaKVStore.bScan(firstKey, lastKey);
+        final List l2 = rheaKVStore.bScan(firstKey, lastKey, false);
+        final List l3 = rheaKVStore.bScan(firstKeyString, lastKeyString);
+        final List l4 = rheaKVStore.bScan(firstKeyString, lastKeyString, false);
+        for (final List l : new List[] { l1, l2, l3, l4 }) {
+            for (final KVEntry kv : l) {
+                LOG.info("sync scan: key={}, value={}", readUtf8(kv.getKey()), readUtf8(kv.getValue()));
+            }
+        }
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server1.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server1.java
new file mode 100644
index 0000000..45287f3
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server1.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.PlacementDriverOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RheaKVStoreOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RocksDBOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.StoreEngineOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Server1 {
+
+    public static void main(final String[] args) {
+        final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured()
+                .withFake(true) // use a fake pd
+                .config();
+        final StoreEngineOptions storeOpts = StoreEngineOptionsConfigured.newConfigured() //
+                .withStorageType(StorageType.RocksDB)
+                .withRocksDBOptions(RocksDBOptionsConfigured.newConfigured().withDbPath(Configs.DB_PATH).config())
+                .withRaftDataPath(Configs.RAFT_DATA_PATH)
+                .withServerAddress(new Endpoint("127.0.0.1", 8181))
+                .config();
+        final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+                .withClusterName(Configs.CLUSTER_NAME) //
+                .withUseParallelCompress(true) //
+                .withInitialServerList(Configs.ALL_NODE_ADDRESSES)
+                .withStoreEngineOptions(storeOpts) //
+                .withPlacementDriverOptions(pdOpts) //
+                .config();
+        System.out.println(opts);
+        final Node node = new Node(opts);
+        node.start();
+        Runtime.getRuntime().addShutdownHook(new Thread(node::stop));
+        System.out.println("server1 start OK");
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server2.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server2.java
new file mode 100644
index 0000000..7823634
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server2.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.PlacementDriverOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RheaKVStoreOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RocksDBOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.StoreEngineOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Server2 {
+
+    public static void main(final String[] args) throws Exception {
+        final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured()
+                .withFake(true) // use a fake pd
+                .config();
+        final StoreEngineOptions storeOpts = StoreEngineOptionsConfigured.newConfigured() //
+                .withStorageType(StorageType.RocksDB)
+                .withRocksDBOptions(RocksDBOptionsConfigured.newConfigured().withDbPath(Configs.DB_PATH).config())
+                .withRaftDataPath(Configs.RAFT_DATA_PATH)
+                .withServerAddress(new Endpoint("127.0.0.1", 8182))
+                .config();
+        final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+                .withClusterName(Configs.CLUSTER_NAME) //
+                .withUseParallelCompress(true) //
+                .withInitialServerList(Configs.ALL_NODE_ADDRESSES)
+                .withStoreEngineOptions(storeOpts) //
+                .withPlacementDriverOptions(pdOpts) //
+                .config();
+        System.out.println(opts);
+        final Node node = new Node(opts);
+        node.start();
+        Runtime.getRuntime().addShutdownHook(new Thread(node::stop));
+        System.out.println("server2 start OK");
+    }
+}
diff --git a/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server3.java b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server3.java
new file mode 100644
index 0000000..13e04ba
--- /dev/null
+++ b/jraft-example/src/main/java/com/alipay/sofa/jraft/example/rheakv/Server3.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.example.rheakv;
+
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.PlacementDriverOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RheaKVStoreOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RocksDBOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.StoreEngineOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Server3 {
+
+    public static void main(final String[] args) throws Exception {
+        final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured()
+                .withFake(true) // use a fake pd
+                .config();
+        final StoreEngineOptions storeOpts = StoreEngineOptionsConfigured.newConfigured() //
+                .withStorageType(StorageType.RocksDB)
+                .withRocksDBOptions(RocksDBOptionsConfigured.newConfigured().withDbPath(Configs.DB_PATH).config())
+                .withRaftDataPath(Configs.RAFT_DATA_PATH)
+                .withServerAddress(new Endpoint("127.0.0.1", 8183))
+                .config();
+        final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+                .withClusterName(Configs.CLUSTER_NAME) //
+                .withUseParallelCompress(true) //
+                .withInitialServerList(Configs.ALL_NODE_ADDRESSES)
+                .withStoreEngineOptions(storeOpts) //
+                .withPlacementDriverOptions(pdOpts) //
+                .config();
+        System.out.println(opts);
+        final Node node = new Node(opts);
+        node.start();
+        Runtime.getRuntime().addShutdownHook(new Thread(node::stop));
+        System.out.println("server3 start OK");
+    }
+}
diff --git a/jraft-example/src/main/resources/conf/rheakv/rheakv_example_client.yaml b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_client.yaml
new file mode 100644
index 0000000..dccf419
--- /dev/null
+++ b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_client.yaml
@@ -0,0 +1,13 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+  # 路由表
+  regionRouteTableOptionsList:
+    - { regionId: -1, initialServerList: "127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183" }
diff --git a/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_1.yaml b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_1.yaml
new file mode 100644
index 0000000..1b7c99e
--- /dev/null
+++ b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_1.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8181
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_2.yaml b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_2.yaml
new file mode 100644
index 0000000..5e5655d
--- /dev/null
+++ b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_2.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8182
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_3.yaml b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_3.yaml
new file mode 100644
index 0000000..a9bafd1
--- /dev/null
+++ b/jraft-example/src/main/resources/conf/rheakv/rheakv_example_node_3.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8183
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/src/main/resources/counter.proto b/jraft-example/src/main/resources/counter.proto
new file mode 100644
index 0000000..94f4af3
--- /dev/null
+++ b/jraft-example/src/main/resources/counter.proto
@@ -0,0 +1,24 @@
+syntax = "proto2";
+
+package jraft;
+
+option java_package = "com.alipay.sofa.jraft.example.counter.rpc";
+option java_outer_classname = "CounterOutter";
+
+
+message GetValueRequest {
+  repeated bool readOnlySafe = 1;
+}
+
+message IncrementAndGetRequest {
+  required int64 delta = 1;
+}
+
+message ValueResponse {
+  required int64 value = 1;
+  required bool success = 2;
+  optional string redirect = 3;
+  optional string errorMsg = 4;
+}
+
+
diff --git a/jraft-example/src/main/resources/log4j2.xml b/jraft-example/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..d62bd94
--- /dev/null
+++ b/jraft-example/src/main/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-example/target/classes/conf/rheakv/rheakv_example_client.yaml b/jraft-example/target/classes/conf/rheakv/rheakv_example_client.yaml
new file mode 100644
index 0000000..dccf419
--- /dev/null
+++ b/jraft-example/target/classes/conf/rheakv/rheakv_example_client.yaml
@@ -0,0 +1,13 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+  # 路由表
+  regionRouteTableOptionsList:
+    - { regionId: -1, initialServerList: "127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183" }
diff --git a/jraft-example/target/classes/conf/rheakv/rheakv_example_node_1.yaml b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_1.yaml
new file mode 100644
index 0000000..1b7c99e
--- /dev/null
+++ b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_1.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8181
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/target/classes/conf/rheakv/rheakv_example_node_2.yaml b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_2.yaml
new file mode 100644
index 0000000..5e5655d
--- /dev/null
+++ b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_2.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8182
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/target/classes/conf/rheakv/rheakv_example_node_3.yaml b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_3.yaml
new file mode 100644
index 0000000..a9bafd1
--- /dev/null
+++ b/jraft-example/target/classes/conf/rheakv/rheakv_example_node_3.yaml
@@ -0,0 +1,25 @@
+##RheaKVStoreOptions
+---
+# 每个store节点包含一个或多个raft-group复制组, 这个字段是所有复制组的名称前缀, 所有的raft-group name遵循
+# [clusterName-regionId]的命名规则
+clusterName: rhea_example
+
+# PD相关选项设置
+placementDriverOptions:
+  # fake==true表示在无PD模式下启动, 无PD模式将失去"自管理"能力, 所有设置都基于当前这个初始的配置文件
+  fake: true
+
+# store存储节点的相关选项设置
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  # raft log存储目录
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    # 端口, 这个是必须配置的选项, 存储层提供rpc服务的监听端口
+    port: 8183
+
+# 集群列表中所有节点的地址列表
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
+
diff --git a/jraft-example/target/classes/counter.proto b/jraft-example/target/classes/counter.proto
new file mode 100644
index 0000000..94f4af3
--- /dev/null
+++ b/jraft-example/target/classes/counter.proto
@@ -0,0 +1,24 @@
+syntax = "proto2";
+
+package jraft;
+
+option java_package = "com.alipay.sofa.jraft.example.counter.rpc";
+option java_outer_classname = "CounterOutter";
+
+
+message GetValueRequest {
+  repeated bool readOnlySafe = 1;
+}
+
+message IncrementAndGetRequest {
+  required int64 delta = 1;
+}
+
+message ValueResponse {
+  required int64 value = 1;
+  required bool success = 2;
+  optional string redirect = 3;
+  optional string errorMsg = 4;
+}
+
+
diff --git a/jraft-example/target/classes/log4j2.xml b/jraft-example/target/classes/log4j2.xml
new file mode 100644
index 0000000..d62bd94
--- /dev/null
+++ b/jraft-example/target/classes/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-extension/pom.xml b/jraft-extension/pom.xml
new file mode 100644
index 0000000..e768bf6
--- /dev/null
+++ b/jraft-extension/pom.xml
@@ -0,0 +1,17 @@
+
+
+    4.0.0
+    
+        jraft-parent
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+
+    jraft-extension
+    pom
+    jraft-extension ${project.version}
+    
+        rpc-grpc-impl
+    
+
+
diff --git a/jraft-extension/rpc-grpc-impl/pom.xml b/jraft-extension/rpc-grpc-impl/pom.xml
new file mode 100644
index 0000000..6c4cae0
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/pom.xml
@@ -0,0 +1,64 @@
+
+
+    4.0.0
+    
+        jraft-extension
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+
+    rpc-grpc-impl
+    jraft-extension ${project.version}
+
+    
+        1.17.0
+    
+
+    
+        
+            ${project.groupId}
+            jraft-core
+        
+        
+        
+            io.grpc
+            grpc-netty-shaded
+            ${io.grpc.version}
+        
+        
+            io.grpc
+            grpc-protobuf
+            ${io.grpc.version}
+        
+        
+            io.grpc
+            grpc-stub
+            ${io.grpc.version}
+        
+        
+            junit
+            junit
+            test
+        
+        
+            junit
+            junit-dep
+            test
+        
+        
+        
+            org.mockito
+            mockito-all
+            test
+        
+        
+        
+            org.openjdk.jmh
+            jmh-core
+        
+        
+            org.openjdk.jmh
+            jmh-generator-annprocess
+        
+    
+
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionInterceptor.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionInterceptor.java
new file mode 100644
index 0000000..cfae0db
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ConnectionInterceptor.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.List;
+
+import io.grpc.Context;
+import io.grpc.Contexts;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.internal.ServerStream;
+import io.grpc.internal.ServerStreamHelper;
+import io.grpc.netty.shaded.io.grpc.netty.NettyConnectionHelper;
+
+import com.alipay.sofa.jraft.rpc.Connection;
+
+/**
+ * Intercepting incoming calls to get {@link Connection} and attach to current {@link Context}
+ * before that are dispatched by {@link ServerCallHandler}.
+ *
+ * @author jiachun.fjc
+ */
+public class ConnectionInterceptor implements ServerInterceptor {
+
+    static final Context.Key STREAM = Context.key("current-stream");
+
+    @Override
+    public  ServerCall.Listener interceptCall(final ServerCall call,
+                                                                 final Metadata headers,
+                                                                 final ServerCallHandler next) {
+        Context ctx = Context.current();
+        final ServerStream stream = ServerStreamHelper.getServerStream(call);
+        if (stream != null) {
+            ctx = ctx.withValue(STREAM, stream);
+        }
+        return Contexts.interceptCall(ctx, call, headers, next);
+    }
+
+    public static Connection getCurrentConnection(final List listeners) {
+        final ServerStream stream = ConnectionInterceptor.STREAM.get();
+        if (stream != null) {
+            return NettyConnectionHelper.getOrCreateConnection(stream, listeners);
+        }
+        return null;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcClient.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcClient.java
new file mode 100644
index 0000000..256611a
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcClient.java
@@ -0,0 +1,358 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import io.grpc.CallOptions;
+import io.grpc.Channel;
+import io.grpc.ConnectivityState;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.MethodDescriptor;
+import io.grpc.protobuf.ProtoUtils;
+import io.grpc.stub.ClientCalls;
+import io.grpc.stub.StreamObserver;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.ReplicatorGroup;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.InvokeTimeoutException;
+import com.alipay.sofa.jraft.error.RemotingException;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.InvokeContext;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.RpcUtils;
+import com.alipay.sofa.jraft.util.DirectExecutor;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.google.protobuf.Message;
+
+/**
+ * GRPC RPC client implement.
+ *
+ * @author nicholas.jxf
+ * @author jiachun.fjc
+ */
+public class GrpcClient implements RpcClient {
+
+    private static final Logger                 LOG                  = LoggerFactory.getLogger(GrpcClient.class);
+
+    private static final int                    RESET_CONN_THRESHOLD = SystemPropertyUtil.getInt(
+                                                                         "jraft.grpc.max.conn.failures.to_reset", 2);
+
+    private final Map managedChannelPool   = new ConcurrentHashMap<>();
+    private final Map  transientFailures    = new ConcurrentHashMap<>();
+    private final Map          parserClasses;
+    private final MarshallerRegistry            marshallerRegistry;
+    private volatile ReplicatorGroup            replicatorGroup;
+
+    public GrpcClient(Map parserClasses, MarshallerRegistry marshallerRegistry) {
+        this.parserClasses = parserClasses;
+        this.marshallerRegistry = marshallerRegistry;
+    }
+
+    @Override
+    public boolean init(final RpcOptions opts) {
+        // do nothing
+        return true;
+    }
+
+    @Override
+    public void shutdown() {
+        closeAllChannels();
+        this.transientFailures.clear();
+    }
+
+    @Override
+    public boolean checkConnection(final Endpoint endpoint) {
+        return checkConnection(endpoint, false);
+    }
+
+    @Override
+    public boolean checkConnection(final Endpoint endpoint, final boolean createIfAbsent) {
+        Requires.requireNonNull(endpoint, "endpoint");
+        return checkChannel(endpoint, createIfAbsent);
+    }
+
+    @Override
+    public void closeConnection(final Endpoint endpoint) {
+        Requires.requireNonNull(endpoint, "endpoint");
+        closeChannel(endpoint);
+    }
+
+    @Override
+    public void registerConnectEventListener(final ReplicatorGroup replicatorGroup) {
+        this.replicatorGroup = replicatorGroup;
+    }
+
+    @Override
+    public Object invokeSync(final Endpoint endpoint, final Object request, final InvokeContext ctx,
+                             final long timeoutMs) throws RemotingException {
+        final CompletableFuture future = new CompletableFuture<>();
+
+        invokeAsync(endpoint, request, ctx, (result, err) -> {
+            if (err == null) {
+                future.complete(result);
+            } else {
+                future.completeExceptionally(err);
+            }
+        }, timeoutMs);
+
+        try {
+            return future.get(timeoutMs, TimeUnit.MILLISECONDS);
+        } catch (final TimeoutException e) {
+            future.cancel(true);
+            throw new InvokeTimeoutException(e);
+        } catch (final Throwable t) {
+            future.cancel(true);
+            throw new RemotingException(t);
+        }
+    }
+
+    @Override
+    public void invokeAsync(final Endpoint endpoint, final Object request, final InvokeContext ctx,
+                            final InvokeCallback callback, final long timeoutMs) {
+        Requires.requireNonNull(endpoint, "endpoint");
+        Requires.requireNonNull(request, "request");
+
+        final Executor executor = callback.executor() != null ? callback.executor() : DirectExecutor.INSTANCE;
+
+        final Channel ch = getCheckedChannel(endpoint);
+        if (ch == null) {
+            executor.execute(() -> callback.complete(null, new RemotingException("Fail to connect: " + endpoint)));
+            return;
+        }
+
+        final MethodDescriptor method = getCallMethod(request);
+        final CallOptions callOpts = CallOptions.DEFAULT.withDeadlineAfter(timeoutMs, TimeUnit.MILLISECONDS);
+
+        ClientCalls.asyncUnaryCall(ch.newCall(method, callOpts), (Message) request, new StreamObserver() {
+
+            @Override
+            public void onNext(final Message value) {
+                executor.execute(() -> callback.complete(value, null));
+            }
+
+            @Override
+            public void onError(final Throwable throwable) {
+                executor.execute(() -> callback.complete(null, throwable));
+            }
+
+            @Override
+            public void onCompleted() {
+                // NO-OP
+            }
+        });
+    }
+
+    private MethodDescriptor getCallMethod(final Object request) {
+        final String interest = request.getClass().getName();
+        final Message reqIns = Requires.requireNonNull(this.parserClasses.get(interest), "null default instance: "
+                                                                                         + interest);
+        return MethodDescriptor //
+            . newBuilder() //
+            .setType(MethodDescriptor.MethodType.UNARY) //
+            .setFullMethodName(MethodDescriptor.generateFullMethodName(interest, GrpcRaftRpcFactory.FIXED_METHOD_NAME)) //
+            .setRequestMarshaller(ProtoUtils.marshaller(reqIns)) //
+            .setResponseMarshaller(
+                ProtoUtils.marshaller(this.marshallerRegistry.findResponseInstanceByRequest(interest))) //
+            .build();
+    }
+
+    private ManagedChannel getCheckedChannel(final Endpoint endpoint) {
+        final ManagedChannel ch = getChannel(endpoint, true);
+
+        if (checkConnectivity(endpoint, ch)) {
+            return ch;
+        }
+
+        return null;
+    }
+
+    private ManagedChannel getChannel(final Endpoint endpoint, final boolean createIfAbsent) {
+        if (createIfAbsent) {
+            return this.managedChannelPool.computeIfAbsent(endpoint, this::newChannel);
+        } else {
+            return this.managedChannelPool.get(endpoint);
+        }
+    }
+
+    private ManagedChannel newChannel(final Endpoint endpoint) {
+        final ManagedChannel ch = ManagedChannelBuilder.forAddress(endpoint.getIp(), endpoint.getPort()) //
+            .usePlaintext() //
+            .directExecutor() //
+            .maxInboundMessageSize(GrpcRaftRpcFactory.RPC_MAX_INBOUND_MESSAGE_SIZE) //
+            .build();
+
+        LOG.info("Creating new channel to: {}.", endpoint);
+
+        // The init channel state is IDLE
+        notifyWhenStateChanged(ConnectivityState.IDLE, endpoint, ch);
+
+        return ch;
+    }
+
+    private ManagedChannel removeChannel(final Endpoint endpoint) {
+        return this.managedChannelPool.remove(endpoint);
+    }
+
+    private void notifyWhenStateChanged(final ConnectivityState state, final Endpoint endpoint, final ManagedChannel ch) {
+        ch.notifyWhenStateChanged(state, () -> onStateChanged(endpoint, ch));
+    }
+
+    private void onStateChanged(final Endpoint endpoint, final ManagedChannel ch) {
+        final ConnectivityState state = ch.getState(false);
+
+        LOG.info("The channel {} is in state: {}.", endpoint, state);
+
+        switch (state) {
+            case READY:
+                notifyReady(endpoint);
+                notifyWhenStateChanged(ConnectivityState.READY, endpoint, ch);
+                break;
+            case TRANSIENT_FAILURE:
+                notifyFailure(endpoint);
+                notifyWhenStateChanged(ConnectivityState.TRANSIENT_FAILURE, endpoint, ch);
+                break;
+            case SHUTDOWN:
+                notifyShutdown(endpoint);
+                break;
+            case CONNECTING:
+                notifyWhenStateChanged(ConnectivityState.CONNECTING, endpoint, ch);
+                break;
+            case IDLE:
+                notifyWhenStateChanged(ConnectivityState.IDLE, endpoint, ch);
+                break;
+        }
+    }
+
+    private void notifyReady(final Endpoint endpoint) {
+        LOG.info("The channel {} has successfully established.", endpoint);
+
+        clearConnFailuresCount(endpoint);
+
+        final ReplicatorGroup rpGroup = this.replicatorGroup;
+        if (rpGroup != null) {
+            try {
+                RpcUtils.runInThread(() -> {
+                    final PeerId peer = new PeerId();
+                    if (peer.parse(endpoint.toString())) {
+                        LOG.info("Peer {} is connected.", peer);
+                        rpGroup.checkReplicator(peer, true);
+                    } else {
+                        LOG.error("Fail to parse peer: {}.", endpoint);
+                    }
+                });
+            } catch (final Throwable t) {
+                LOG.error("Fail to check replicator {}.", endpoint, t);
+            }
+        }
+    }
+
+    private void notifyFailure(final Endpoint endpoint) {
+        LOG.warn("There has been some transient failure on this channel {}.", endpoint);
+    }
+
+    private void notifyShutdown(final Endpoint endpoint) {
+        LOG.warn("This channel {} has started shutting down. Any new RPCs should fail immediately.", endpoint);
+    }
+
+    private void closeAllChannels() {
+        for (final Map.Entry entry : this.managedChannelPool.entrySet()) {
+            final ManagedChannel ch = entry.getValue();
+            LOG.info("Shutdown managed channel: {}, {}.", entry.getKey(), ch);
+            ManagedChannelHelper.shutdownAndAwaitTermination(ch);
+        }
+        this.managedChannelPool.clear();
+    }
+
+    private void closeChannel(final Endpoint endpoint) {
+        final ManagedChannel ch = removeChannel(endpoint);
+        LOG.info("Close connection: {}, {}.", endpoint, ch);
+        if (ch != null) {
+            ManagedChannelHelper.shutdownAndAwaitTermination(ch);
+        }
+    }
+
+    private boolean checkChannel(final Endpoint endpoint, final boolean createIfAbsent) {
+        final ManagedChannel ch = getChannel(endpoint, createIfAbsent);
+
+        if (ch == null) {
+            return false;
+        }
+
+        return checkConnectivity(endpoint, ch);
+    }
+
+    private int incConnFailuresCount(final Endpoint endpoint) {
+        return this.transientFailures.computeIfAbsent(endpoint, ep -> new AtomicInteger()).incrementAndGet();
+    }
+
+    private void clearConnFailuresCount(final Endpoint endpoint) {
+        this.transientFailures.remove(endpoint);
+    }
+
+    private boolean checkConnectivity(final Endpoint endpoint, final ManagedChannel ch) {
+        final ConnectivityState st = ch.getState(false);
+
+        if (st != ConnectivityState.TRANSIENT_FAILURE && st != ConnectivityState.SHUTDOWN) {
+            return true;
+        }
+
+        final int c = incConnFailuresCount(endpoint);
+        if (c < RESET_CONN_THRESHOLD) {
+            if (c == RESET_CONN_THRESHOLD - 1) {
+                // For sub-channels that are in TRANSIENT_FAILURE state, short-circuit the backoff timer and make
+                // them reconnect immediately. May also attempt to invoke NameResolver#refresh
+                ch.resetConnectBackoff();
+            }
+            return true;
+        }
+
+        clearConnFailuresCount(endpoint);
+
+        final ManagedChannel removedCh = removeChannel(endpoint);
+
+        if (removedCh == null) {
+            // The channel has been removed and closed by another
+            return false;
+        }
+
+        LOG.warn("Channel[{}] in [INACTIVE] state {} times, it has been removed from the pool.", endpoint, c);
+
+        if (removedCh != ch) {
+            // Now that it's removed, close it
+            ManagedChannelHelper.shutdownAndAwaitTermination(removedCh, 100);
+        }
+
+        ManagedChannelHelper.shutdownAndAwaitTermination(ch, 100);
+
+        return false;
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcRaftRpcFactory.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcRaftRpcFactory.java
new file mode 100644
index 0000000..dccea25
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcRaftRpcFactory.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.util.MutableHandlerRegistry;
+
+import com.alipay.sofa.jraft.rpc.RaftRpcFactory;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.RpcResponseFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.SPI;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.google.protobuf.Message;
+
+/**
+ * @author nicholas.jxf
+ * @author jiachun.fjc
+ */
+@SPI(priority = 1)
+public class GrpcRaftRpcFactory implements RaftRpcFactory {
+
+    static final String             FIXED_METHOD_NAME              = "_call";
+    static final int                RPC_SERVER_PROCESSOR_POOL_SIZE = SystemPropertyUtil
+                                                                       .getInt(
+                                                                           "jraft.grpc.default_rpc_server_processor_pool_size",
+                                                                           100);
+
+    static final int                RPC_MAX_INBOUND_MESSAGE_SIZE   = SystemPropertyUtil.getInt(
+                                                                       "jraft.grpc.max_inbound_message_size.bytes",
+                                                                       4 * 1024 * 1024);
+
+    static final RpcResponseFactory RESPONSE_FACTORY               = new GrpcResponseFactory();
+
+    final Map      parserClasses                  = new ConcurrentHashMap<>();
+    final MarshallerRegistry        defaultMarshallerRegistry      = new MarshallerRegistry() {
+
+                                                                       @Override
+                                                                       public Message findResponseInstanceByRequest(final String reqCls) {
+                                                                           return MarshallerHelper
+                                                                               .findRespInstance(reqCls);
+                                                                       }
+
+                                                                       @Override
+                                                                       public void registerResponseInstance(final String reqCls,
+                                                                                                            final Message respIns) {
+                                                                           MarshallerHelper.registerRespInstance(
+                                                                               reqCls, respIns);
+                                                                       }
+                                                                   };
+
+    @Override
+    public void registerProtobufSerializer(final String className, final Object... args) {
+        this.parserClasses.put(className, (Message) args[0]);
+    }
+
+    @Override
+    public RpcClient createRpcClient(final ConfigHelper helper) {
+        final RpcClient rpcClient = new GrpcClient(this.parserClasses, getMarshallerRegistry());
+        if (helper != null) {
+            helper.config(rpcClient);
+        }
+        return rpcClient;
+    }
+
+    @Override
+    public RpcServer createRpcServer(final Endpoint endpoint, final ConfigHelper helper) {
+        final int port = Requires.requireNonNull(endpoint, "endpoint").getPort();
+        Requires.requireTrue(port > 0 && port < 0xFFFF, "port out of range:" + port);
+        final MutableHandlerRegistry handlerRegistry = new MutableHandlerRegistry();
+        final Server server = ServerBuilder.forPort(port) //
+            .fallbackHandlerRegistry(handlerRegistry) //
+            .directExecutor() //
+            .maxInboundMessageSize(RPC_MAX_INBOUND_MESSAGE_SIZE) //
+            .build();
+        final RpcServer rpcServer = new GrpcServer(server, handlerRegistry, this.parserClasses, getMarshallerRegistry());
+        if (helper != null) {
+            helper.config(rpcServer);
+        }
+        return rpcServer;
+    }
+
+    @Override
+    public RpcResponseFactory getRpcResponseFactory() {
+        return RESPONSE_FACTORY;
+    }
+
+    @Override
+    public boolean isReplicatorPipelineEnabled() {
+        return true;
+    }
+
+    public MarshallerRegistry getMarshallerRegistry() {
+        return defaultMarshallerRegistry;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcResponseFactory.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcResponseFactory.java
new file mode 100644
index 0000000..3f15d26
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcResponseFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcResponseFactory;
+import com.alipay.sofa.jraft.util.Requires;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.Message;
+
+/**
+ * Helper to create error response for GRPC implementation.
+ *
+ * @author jiachun.fjc
+ */
+public class GrpcResponseFactory implements RpcResponseFactory {
+
+    @Override
+    public Message newResponse(final Message parent, final int code, final String fmt, final Object... args) {
+        final RpcRequests.ErrorResponse.Builder eBuilder = RpcRequests.ErrorResponse.newBuilder();
+        eBuilder.setErrorCode(code);
+        if (fmt != null) {
+            eBuilder.setErrorMsg(String.format(fmt, args));
+        }
+
+        if (parent == null || parent instanceof RpcRequests.ErrorResponse) {
+            return eBuilder.build();
+        }
+
+        final Descriptors.FieldDescriptor errFd = parent //
+            .getDescriptorForType() //
+            .findFieldByNumber(ERROR_RESPONSE_NUM);
+        Requires.requireNonNull(errFd, "errFd");
+        final Message.Builder builder = parent.toBuilder();
+        for (final Descriptors.FieldDescriptor fd : parent.getDescriptorForType().getFields()) {
+            builder.setField(fd, parent.getField(fd));
+        }
+        return builder //
+            .setField(errFd, eBuilder.build()) //
+            .build();
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServer.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServer.java
new file mode 100644
index 0000000..45cf7fd
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServer.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.io.IOException;
+import java.net.SocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import io.grpc.MethodDescriptor;
+import io.grpc.Server;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.ServerInterceptors;
+import io.grpc.ServerServiceDefinition;
+import io.grpc.protobuf.ProtoUtils;
+import io.grpc.stub.ServerCalls;
+import io.grpc.util.MutableHandlerRegistry;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.google.protobuf.Message;
+
+/**
+ * GRPC RPC server implement.
+ *
+ * @author nicholas.jxf
+ * @author jiachun.fjc
+ */
+public class GrpcServer implements RpcServer {
+
+    private static final Logger                       LOG                  = LoggerFactory.getLogger(GrpcServer.class);
+
+    private static final String                       EXECUTOR_NAME        = "grpc-default-executor";
+
+    private final Server                              server;
+    private final MutableHandlerRegistry              handlerRegistry;
+    private final Map                parserClasses;
+    private final MarshallerRegistry                  marshallerRegistry;
+    private final List             serverInterceptors   = new CopyOnWriteArrayList<>();
+    private final List closedEventListeners = new CopyOnWriteArrayList<>();
+    private final AtomicBoolean                       started              = new AtomicBoolean(false);
+
+    private ExecutorService                           defaultExecutor;
+
+    public GrpcServer(Server server, MutableHandlerRegistry handlerRegistry, Map parserClasses,
+                      MarshallerRegistry marshallerRegistry) {
+        this.server = server;
+        this.handlerRegistry = handlerRegistry;
+        this.parserClasses = parserClasses;
+        this.marshallerRegistry = marshallerRegistry;
+        registerDefaultServerInterceptor();
+    }
+
+    @Override
+    public boolean init(final Void opts) {
+        if (!this.started.compareAndSet(false, true)) {
+            throw new IllegalStateException("grpc server has started");
+        }
+
+        this.defaultExecutor = ThreadPoolUtil.newBuilder() //
+            .poolName(EXECUTOR_NAME) //
+            .enableMetric(true) //
+            .coreThreads(Math.min(20, GrpcRaftRpcFactory.RPC_SERVER_PROCESSOR_POOL_SIZE / 5)) //
+            .maximumThreads(GrpcRaftRpcFactory.RPC_SERVER_PROCESSOR_POOL_SIZE) //
+            .keepAliveSeconds(60L) //
+            .workQueue(new SynchronousQueue<>()) //
+            .threadFactory(new NamedThreadFactory(EXECUTOR_NAME + "-", true)) //
+            .rejectedHandler((r, executor) -> {
+                throw new RejectedExecutionException("[" + EXECUTOR_NAME + "], task " + r.toString() +
+                        " rejected from " +
+                        executor.toString());
+            })
+            .build();
+
+        try {
+            this.server.start();
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        }
+        return true;
+    }
+
+    @Override
+    public void shutdown() {
+        if (!this.started.compareAndSet(true, false)) {
+            return;
+        }
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.defaultExecutor);
+        GrpcServerHelper.shutdownAndAwaitTermination(this.server);
+    }
+
+    @Override
+    public void registerConnectionClosedEventListener(final ConnectionClosedEventListener listener) {
+        this.closedEventListeners.add(listener);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void registerProcessor(final RpcProcessor processor) {
+        final String interest = processor.interest();
+        final Message reqIns = Requires.requireNonNull(this.parserClasses.get(interest), "null default instance: " + interest);
+        final MethodDescriptor method = MethodDescriptor //
+                .newBuilder() //
+                .setType(MethodDescriptor.MethodType.UNARY) //
+                .setFullMethodName(
+                    MethodDescriptor.generateFullMethodName(processor.interest(), GrpcRaftRpcFactory.FIXED_METHOD_NAME)) //
+                .setRequestMarshaller(ProtoUtils.marshaller(reqIns)) //
+                .setResponseMarshaller(ProtoUtils.marshaller(this.marshallerRegistry.findResponseInstanceByRequest(interest))) //
+                .build();
+
+        final ServerCallHandler handler = ServerCalls.asyncUnaryCall(
+                (request, responseObserver) -> {
+                    final SocketAddress remoteAddress = RemoteAddressInterceptor.getRemoteAddress();
+                    final Connection conn = ConnectionInterceptor.getCurrentConnection(this.closedEventListeners);
+
+                    final RpcContext rpcCtx = new RpcContext() {
+
+                        @Override
+                        public void sendResponse(final Object responseObj) {
+                            try {
+                                responseObserver.onNext((Message) responseObj);
+                                responseObserver.onCompleted();
+                            } catch (final Throwable t) {
+                                LOG.warn("[GRPC] failed to send response.", t);
+                            }
+                        }
+
+                        @Override
+                        public Connection getConnection() {
+                            if (conn == null) {
+                                throw new IllegalStateException("fail to get connection");
+                            }
+                            return conn;
+                        }
+
+                        @Override
+                        public String getRemoteAddress() {
+                            // Rely on GRPC's capabilities, not magic (netty channel)
+                            return remoteAddress != null ? remoteAddress.toString() : null;
+                        }
+                    };
+
+                    final RpcProcessor.ExecutorSelector selector = processor.executorSelector();
+                    Executor executor;
+                    if (selector != null && request instanceof RpcRequests.AppendEntriesRequest) {
+                        final RpcRequests.AppendEntriesRequest req = (RpcRequests.AppendEntriesRequest) request;
+                        final RpcRequests.AppendEntriesRequestHeader.Builder header = RpcRequests.AppendEntriesRequestHeader //
+                                .newBuilder() //
+                                .setGroupId(req.getGroupId()) //
+                                .setPeerId(req.getPeerId()) //
+                                .setServerId(req.getServerId());
+                        executor = selector.select(interest, header.build());
+                    } else {
+                        executor = processor.executor();
+                    }
+
+                    if (executor == null) {
+                        executor = this.defaultExecutor;
+                    }
+
+                    if (executor != null) {
+                        executor.execute(() -> processor.handleRequest(rpcCtx, request));
+                    } else {
+                        processor.handleRequest(rpcCtx, request);
+                    }
+                });
+
+        final ServerServiceDefinition serviceDef = ServerServiceDefinition //
+                .builder(interest) //
+                .addMethod(method, handler) //
+                .build();
+
+        this.handlerRegistry
+            .addService(ServerInterceptors.intercept(serviceDef, this.serverInterceptors.toArray(new ServerInterceptor[0])));
+    }
+
+    @Override
+    public int boundPort() {
+        return this.server.getPort();
+    }
+
+    public void setDefaultExecutor(ExecutorService defaultExecutor) {
+        this.defaultExecutor = defaultExecutor;
+    }
+
+    public Server getServer() {
+        return server;
+    }
+
+    public MutableHandlerRegistry getHandlerRegistry() {
+        return handlerRegistry;
+    }
+
+    public boolean addServerInterceptor(final ServerInterceptor interceptor) {
+        return this.serverInterceptors.add(interceptor);
+    }
+
+    private void registerDefaultServerInterceptor() {
+        this.serverInterceptors.add(new RemoteAddressInterceptor());
+        this.serverInterceptors.add(new ConnectionInterceptor());
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServerHelper.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServerHelper.java
new file mode 100644
index 0000000..2dc44ce
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/GrpcServerHelper.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.concurrent.TimeUnit;
+
+import io.grpc.Server;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GrpcServerHelper {
+
+    private static final Logger LOG = LoggerFactory.getLogger(GrpcServerHelper.class);
+
+    /**
+     * @see #shutdownAndAwaitTermination(Server, long)
+     */
+    public static boolean shutdownAndAwaitTermination(final Server server) {
+        return shutdownAndAwaitTermination(server, 1000);
+    }
+
+    /**
+     * The following method shuts down an {@code Server} in two
+     * phases, first by calling {@code shutdown} to reject incoming tasks,
+     * and then calling {@code shutdownNow}, if necessary, to cancel any
+     * lingering tasks.
+     */
+    public static boolean shutdownAndAwaitTermination(final Server server, final long timeoutMillis) {
+        if (server == null) {
+            return true;
+        }
+        // disable new tasks from being submitted
+        server.shutdown();
+        final TimeUnit unit = TimeUnit.MILLISECONDS;
+        final long phaseOne = timeoutMillis / 5;
+        try {
+            // wait a while for existing tasks to terminate
+            if (server.awaitTermination(phaseOne, unit)) {
+                return true;
+            }
+            server.shutdownNow();
+            // wait a while for tasks to respond to being cancelled
+            if (server.awaitTermination(timeoutMillis - phaseOne, unit)) {
+                return true;
+            }
+            LOG.warn("Fail to shutdown grpc server: {}.", server);
+        } catch (final InterruptedException e) {
+            // (Re-)cancel if current thread also interrupted
+            server.shutdownNow();
+            // preserve interrupt status
+            Thread.currentThread().interrupt();
+        }
+        return false;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ManagedChannelHelper.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ManagedChannelHelper.java
new file mode 100644
index 0000000..fe48cac
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/ManagedChannelHelper.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.concurrent.TimeUnit;
+
+import io.grpc.ManagedChannel;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ManagedChannelHelper {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ManagedChannelHelper.class);
+
+    /**
+     * @see #shutdownAndAwaitTermination(ManagedChannel, long)
+     */
+    public static boolean shutdownAndAwaitTermination(final ManagedChannel mChannel) {
+        return shutdownAndAwaitTermination(mChannel, 1000);
+    }
+
+    /**
+     * The following method shuts down an {@code ManagedChannel} in two
+     * phases, first by calling {@code shutdown} to reject incoming tasks,
+     * and then calling {@code shutdownNow}, if necessary, to cancel any
+     * lingering tasks.
+     */
+    public static boolean shutdownAndAwaitTermination(final ManagedChannel mChannel, final long timeoutMillis) {
+        if (mChannel == null) {
+            return true;
+        }
+        // disable new tasks from being submitted
+        mChannel.shutdown();
+        final TimeUnit unit = TimeUnit.MILLISECONDS;
+        final long phaseOne = timeoutMillis / 5;
+        try {
+            // wait a while for existing tasks to terminate
+            if (mChannel.awaitTermination(phaseOne, unit)) {
+                return true;
+            }
+            mChannel.shutdownNow();
+            // wait a while for tasks to respond to being cancelled
+            if (mChannel.awaitTermination(timeoutMillis - phaseOne, unit)) {
+                return true;
+            }
+            LOG.warn("Fail to shutdown managed channel: {}.", mChannel);
+        } catch (final InterruptedException e) {
+            // (Re-)cancel if current thread also interrupted
+            mChannel.shutdownNow();
+            // preserve interrupt status
+            Thread.currentThread().interrupt();
+        }
+        return false;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerHelper.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerHelper.java
new file mode 100644
index 0000000..4d0ba80
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerHelper.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.rpc.CliRequests;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.google.protobuf.Message;
+
+/**
+ * Not thread safe.
+ *
+ * @author jiachun.fjc
+ */
+public class MarshallerHelper {
+
+    private static Map messages = new ConcurrentHashMap<>();
+
+    static {
+        messages.put(RpcRequests.AppendEntriesRequest.class.getName(),
+            RpcRequests.AppendEntriesResponse.getDefaultInstance());
+        messages.put(RpcRequests.GetFileRequest.class.getName(), RpcRequests.GetFileResponse.getDefaultInstance());
+        messages.put(RpcRequests.InstallSnapshotRequest.class.getName(),
+            RpcRequests.InstallSnapshotResponse.getDefaultInstance());
+        messages.put(RpcRequests.RequestVoteRequest.class.getName(),
+            RpcRequests.RequestVoteResponse.getDefaultInstance());
+        messages.put(RpcRequests.PingRequest.class.getName(), RpcRequests.ErrorResponse.getDefaultInstance());
+        messages
+            .put(RpcRequests.TimeoutNowRequest.class.getName(), RpcRequests.TimeoutNowResponse.getDefaultInstance());
+        messages.put(RpcRequests.ReadIndexRequest.class.getName(), RpcRequests.ReadIndexResponse.getDefaultInstance());
+        messages.put(CliRequests.AddPeerRequest.class.getName(), CliRequests.AddPeerResponse.getDefaultInstance());
+        messages
+            .put(CliRequests.RemovePeerRequest.class.getName(), CliRequests.RemovePeerResponse.getDefaultInstance());
+        messages.put(CliRequests.ResetPeerRequest.class.getName(), RpcRequests.ErrorResponse.getDefaultInstance());
+        messages.put(CliRequests.ChangePeersRequest.class.getName(),
+            CliRequests.ChangePeersResponse.getDefaultInstance());
+        messages.put(CliRequests.GetLeaderRequest.class.getName(), CliRequests.GetLeaderResponse.getDefaultInstance());
+        messages.put(CliRequests.SnapshotRequest.class.getName(), RpcRequests.ErrorResponse.getDefaultInstance());
+        messages.put(CliRequests.TransferLeaderRequest.class.getName(), RpcRequests.ErrorResponse.getDefaultInstance());
+        messages.put(CliRequests.GetPeersRequest.class.getName(), CliRequests.GetPeersResponse.getDefaultInstance());
+        messages.put(CliRequests.AddLearnersRequest.class.getName(),
+            CliRequests.LearnersOpResponse.getDefaultInstance());
+        messages.put(CliRequests.RemoveLearnersRequest.class.getName(),
+            CliRequests.LearnersOpResponse.getDefaultInstance());
+        messages.put(CliRequests.ResetLearnersRequest.class.getName(),
+            CliRequests.LearnersOpResponse.getDefaultInstance());
+    }
+
+    public static Message findRespInstance(final String name) {
+        return messages.get(name);
+    }
+
+    public static void registerRespInstance(final String name, final Message instance) {
+        messages.put(name, instance);
+    }
+}
\ No newline at end of file
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerRegistry.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerRegistry.java
new file mode 100644
index 0000000..cb98eb7
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/MarshallerRegistry.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import com.google.protobuf.Message;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface MarshallerRegistry {
+
+    /**
+     * Find response default instance by request's class name.
+     *
+     * @param reqCls request class name
+     * @return response default instance
+     */
+    Message findResponseInstanceByRequest(final String reqCls);
+
+    /**
+     * Register response default instance.
+     *
+     * @param reqCls  request class name
+     * @param respIns response default instance
+     */
+    void registerResponseInstance(final String reqCls, final Message respIns);
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/RemoteAddressInterceptor.java b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/RemoteAddressInterceptor.java
new file mode 100644
index 0000000..46b7817
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/com/alipay/sofa/jraft/rpc/impl/RemoteAddressInterceptor.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import java.net.SocketAddress;
+
+import io.grpc.Context;
+import io.grpc.Contexts;
+import io.grpc.Grpc;
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+
+/**
+ * GRPC server interceptor to trace remote address.
+ *
+ * @author nicholas.jxf
+ */
+public class RemoteAddressInterceptor implements ServerInterceptor {
+
+    private static final Context.Key REMOTE_ADDRESS = Context.key("remote-address");
+
+    @Override
+    public  ServerCall.Listener interceptCall(final ServerCall call,
+                                                                 final Metadata headers,
+                                                                 final ServerCallHandler next) {
+        final Context ctx = Context.current() //
+            .withValue(REMOTE_ADDRESS, call.getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
+        return Contexts.interceptCall(ctx, call, headers, next);
+    }
+
+    public static SocketAddress getRemoteAddress() {
+        return REMOTE_ADDRESS.get();
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/internal/ServerStreamHelper.java b/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/internal/ServerStreamHelper.java
new file mode 100644
index 0000000..73abb41
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/internal/ServerStreamHelper.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.grpc.internal;
+
+import io.grpc.ServerCall;
+
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+
+/**
+ * Get grpc's server stream.
+ *
+ * @author jiachun.fjc
+ */
+public class ServerStreamHelper {
+
+    private static final ReferenceFieldUpdater, ServerStream> STREAM_GETTER = Updaters
+                                                                                                     .newReferenceFieldUpdater(
+                                                                                                         ServerCallImpl.class,
+                                                                                                         "stream");
+
+    public static ServerStream getServerStream(final ServerCall call) {
+        if (call instanceof ServerCallImpl) {
+            return STREAM_GETTER.get((ServerCallImpl) call);
+        }
+        return null;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/netty/shaded/io/grpc/netty/NettyConnectionHelper.java b/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/netty/shaded/io/grpc/netty/NettyConnectionHelper.java
new file mode 100644
index 0000000..93ccb04
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/java/io/grpc/netty/shaded/io/grpc/netty/NettyConnectionHelper.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.grpc.netty.shaded.io.grpc.netty;
+
+import java.util.List;
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.impl.ConnectionClosedEventListener;
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+import io.grpc.internal.ServerStream;
+import io.grpc.netty.shaded.io.netty.channel.Channel;
+import io.grpc.netty.shaded.io.netty.util.Attribute;
+import io.grpc.netty.shaded.io.netty.util.AttributeKey;
+
+/**
+ * Get netty channel.
+ *
+ * @author jiachun.fjc
+ */
+public class NettyConnectionHelper {
+
+    private static final ReferenceFieldUpdater CHANNEL_GETTER = Updaters
+                                                                                              .newReferenceFieldUpdater(
+                                                                                                  NettyServerStream.class,
+                                                                                                  "channel");
+
+    private static final AttributeKey                     NETTY_CONN_KEY = AttributeKey
+                                                                                              .valueOf("netty.conn");
+
+    public static Connection getOrCreateConnection(final ServerStream stream,
+                                                   final List listeners) {
+        if (stream instanceof NettyServerStream) {
+            return attachChannel(CHANNEL_GETTER.get((NettyServerStream) stream), listeners);
+        }
+        return null;
+    }
+
+    private static Connection attachChannel(final Channel channel, final List listeners) {
+        if (channel == null) {
+            return null;
+        }
+
+        final Attribute attr = channel.attr(NETTY_CONN_KEY);
+        NettyConnection conn = attr.get();
+        if (conn == null) {
+            final NettyConnection newConn = new NettyConnection(channel);
+            conn = attr.setIfAbsent(newConn);
+            if (conn == null) {
+                conn = newConn;
+                for (final ConnectionClosedEventListener l : listeners) {
+                    conn.addClosedEventListener(l);
+                }
+            }
+        }
+
+        return conn;
+    }
+}
+
+class NettyConnection implements Connection {
+
+    private final Channel ch;
+
+    NettyConnection(final Channel ch) {
+        this.ch = ch;
+    }
+
+    @Override
+    public Object setAttributeIfAbsent(final String key, final Object value) {
+        return this.ch.attr(AttributeKey.valueOf(key)).setIfAbsent(value);
+    }
+
+    @Override
+    public Object getAttribute(final String key) {
+        return this.ch.attr(AttributeKey.valueOf(key)).get();
+    }
+
+    @Override
+    public void setAttribute(final String key, final Object value) {
+        this.ch.attr(AttributeKey.valueOf(key)).set(value);
+    }
+
+    @Override
+    public void close() {
+        this.ch.close();
+    }
+
+    void addClosedEventListener(final ConnectionClosedEventListener listener) {
+      this.ch.closeFuture() //
+      .addListener(
+          future -> listener.onClosed(this.ch.remoteAddress().toString(), NettyConnection.this));
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory b/jraft-extension/rpc-grpc-impl/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
new file mode 100644
index 0000000..35e8271
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rpc.RaftRpcFactory
@@ -0,0 +1 @@
+com.alipay.sofa.jraft.rpc.impl.GrpcRaftRpcFactory
\ No newline at end of file
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
new file mode 100644
index 0000000..1995d3b
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/RouteTableTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.NodeImpl;
+import com.alipay.sofa.jraft.core.TestCluster;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public class RouteTableTest {
+
+    static final Logger  LOG     = LoggerFactory.getLogger(RouteTableTest.class);
+
+    private String       dataPath;
+
+    private TestCluster  cluster;
+    private final String groupId = "RouteTableTest";
+
+    CliClientServiceImpl cliClientService;
+
+    @Before
+    public void setup() throws Exception {
+        cliClientService = new CliClientServiceImpl();
+        cliClientService.init(new CliOptions());
+        this.dataPath = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.dataPath));
+        assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        final List peers = TestUtils.generatePeers(3);
+
+        cluster = new TestCluster(groupId, dataPath, peers);
+        for (final PeerId peer : peers) {
+            cluster.start(peer.getEndpoint());
+        }
+        cluster.waitLeader();
+    }
+
+    @After
+    public void teardown() throws Exception {
+        cliClientService.shutdown();
+        cluster.stopAll();
+        if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+            Thread.sleep(1000);
+            assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        }
+        FileUtils.deleteDirectory(new File(this.dataPath));
+        NodeManager.getInstance().clear();
+        RouteTable.getInstance().reset();
+    }
+
+    @Test
+    public void testUpdateConfSelectLeader() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        assertNull(rt.getConfiguration(groupId));
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        assertEquals(rt.getConfiguration(groupId), new Configuration(cluster.getPeers()));
+        assertNull(rt.selectLeader(groupId));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+        final PeerId leader = rt.selectLeader(groupId);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testUpdateLeaderNull() throws Exception {
+        this.testUpdateConfSelectLeader();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateLeader(groupId, (PeerId) null);
+        assertNull(rt.selectLeader(groupId));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+
+        final PeerId leader = rt.selectLeader(groupId);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testRefreshLeaderWhenLeaderStops() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        testUpdateConfSelectLeader();
+        PeerId leader = rt.selectLeader(groupId);
+        this.cluster.stop(leader.getEndpoint());
+        this.cluster.waitLeader();
+        final PeerId oldLeader = leader.copy();
+
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+        leader = rt.selectLeader(groupId);
+        assertNotEquals(leader, oldLeader);
+        assertEquals(leader, cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @Test
+    public void testRefreshLeaderWhenFirstPeerDown() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+        cluster.stop(cluster.getPeers().get(0).getEndpoint());
+        Thread.sleep(1000);
+        this.cluster.waitLeader();
+        assertTrue(rt.refreshLeader(cliClientService, groupId, 10000).isOk());
+    }
+
+    @Test
+    public void testRefreshFail() throws Exception {
+        cluster.stopAll();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        final Status status = rt.refreshLeader(cliClientService, groupId, 5000);
+        assertFalse(status.isOk());
+        assertTrue(status.getErrorMsg().contains("Fail to init channel"));
+    }
+
+    @Test
+    public void testRefreshConfiguration() throws Exception {
+        final RouteTable rt = RouteTable.getInstance();
+        final List partConf = new ArrayList<>();
+        partConf.add(cluster.getLeader().getLeaderId());
+        // part of peers conf, only contains leader peer
+        rt.updateConfiguration(groupId, new Configuration(partConf));
+        // fetch all conf
+        final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+        assertTrue(st.isOk());
+        final Configuration newCnf = rt.getConfiguration(groupId);
+        assertArrayEquals(new HashSet<>(cluster.getPeers()).toArray(), new HashSet<>(newCnf.getPeerSet()).toArray());
+    }
+
+    @Test
+    public void testRefreshConfigurationFail() throws Exception {
+        cluster.stopAll();
+        final RouteTable rt = RouteTable.getInstance();
+        rt.updateConfiguration(groupId, new Configuration(cluster.getPeers()));
+        final Status st = rt.refreshConfiguration(cliClientService, groupId, 10000);
+        assertFalse(st.isOk());
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
new file mode 100644
index 0000000..9646f88
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/CliServiceTest.java
@@ -0,0 +1,493 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.CliService;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class CliServiceTest {
+
+    private String           dataPath;
+
+    private TestCluster      cluster;
+    private final String     groupId           = "CliServiceTest";
+
+    private CliService       cliService;
+
+    private Configuration    conf;
+
+    @Rule
+    public TestName          testName          = new TestName();
+
+    private static final int LEARNER_PORT_STEP = 100;
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+        this.dataPath = TestUtils.mkTempDir();
+        FileUtils.forceMkdir(new File(this.dataPath));
+        assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        final List peers = TestUtils.generatePeers(3);
+
+        final LinkedHashSet learners = new LinkedHashSet<>();
+        //2 learners
+        for (int i = 0; i < 2; i++) {
+            learners.add(new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + i));
+        }
+
+        this.cluster = new TestCluster(this.groupId, this.dataPath, peers, learners, 300);
+        for (final PeerId peer : peers) {
+            this.cluster.start(peer.getEndpoint());
+        }
+
+        for (final PeerId peer : learners) {
+            this.cluster.startLearner(peer);
+        }
+
+        this.cluster.waitLeader();
+
+        this.cliService = new CliServiceImpl();
+        this.conf = new Configuration(peers, learners);
+        assertTrue(this.cliService.init(new CliOptions()));
+    }
+
+    @After
+    public void teardown() throws Exception {
+        this.cliService.shutdown();
+        this.cluster.stopAll();
+        if (NodeImpl.GLOBAL_NUM_NODES.get() > 0) {
+            Thread.sleep(1000);
+            assertEquals(NodeImpl.GLOBAL_NUM_NODES.get(), 0);
+        }
+        FileUtils.deleteDirectory(new File(this.dataPath));
+        NodeManager.getInstance().clear();
+        RouteTable.getInstance().reset();
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+    }
+
+    @Test
+    public void testTransferLeader() throws Exception {
+        final PeerId leader = this.cluster.getLeader().getNodeId().getPeerId().copy();
+        assertNotNull(leader);
+
+        final Set peers = this.conf.getPeerSet();
+        PeerId targetPeer = null;
+        for (final PeerId peer : peers) {
+            if (!peer.equals(leader)) {
+                targetPeer = peer;
+                break;
+            }
+        }
+        assertNotNull(targetPeer);
+        assertTrue(this.cliService.transferLeader(this.groupId, this.conf, targetPeer).isOk());
+        this.cluster.waitLeader();
+        assertEquals(targetPeer, this.cluster.getLeader().getNodeId().getPeerId());
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private void sendTestTaskAndWait(final Node node, final int code) throws InterruptedException {
+        final CountDownLatch latch = new CountDownLatch(10);
+        for (int i = 0; i < 10; i++) {
+            final ByteBuffer data = ByteBuffer.wrap(("hello" + i).getBytes());
+            final Task task = new Task(data, new ExpectClosure(code, null, latch));
+            node.apply(task);
+        }
+        assertTrue(latch.await(10, TimeUnit.SECONDS));
+    }
+
+    @Test
+    public void testLearnerServices() throws Exception {
+        final PeerId learner3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + LEARNER_PORT_STEP + 3);
+        assertTrue(this.cluster.startLearner(learner3));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(500);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+                assertEquals(10, fsm.getLogs().size());
+            }
+        }
+        assertEquals(0, this.cluster.getFsmByPeer(learner3).getLogs().size());
+        List oldLearners = new ArrayList(this.conf.getLearners());
+        assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Add learner3
+        this.cliService.addLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        Thread.sleep(100);
+        assertEquals(10, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(500);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+
+        }
+        List newLearners = new ArrayList<>(oldLearners);
+        newLearners.add(learner3);
+        assertEquals(newLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(newLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Remove  3
+        this.cliService.removeLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(500);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!fsm.getAddress().equals(learner3.getEndpoint())) {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+        // Latest 10 logs are not replicated to learner3, because it's removed.
+        assertEquals(20, this.cluster.getFsmByPeer(learner3).getLogs().size());
+        assertEquals(oldLearners, this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(oldLearners, this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Set learners into [learner3]
+        this.cliService.resetLearners(this.groupId, this.conf, Arrays.asList(learner3));
+        Thread.sleep(100);
+        assertEquals(30, this.cluster.getFsmByPeer(learner3).getLogs().size());
+
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(500);
+        // Latest 10 logs are not replicated to learner1 and learner2, because they were removed by resetting learners set.
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (!oldLearners.contains(new PeerId(fsm.getAddress(), 0))) {
+                assertEquals(40, fsm.getLogs().size());
+            } else {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+        assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+        assertEquals(Arrays.asList(learner3), this.cliService.getAliveLearners(this.groupId, this.conf));
+
+        // Stop learner3
+        this.cluster.stop(learner3.getEndpoint());
+        Thread.sleep(1000);
+        assertEquals(Arrays.asList(learner3), this.cliService.getLearners(this.groupId, this.conf));
+        assertTrue(this.cliService.getAliveLearners(this.groupId, this.conf).isEmpty());
+    }
+
+    @Test
+    public void testAddPeerRemovePeer() throws Exception {
+        final PeerId peer3 = new PeerId(TestUtils.getMyIp(), TestUtils.INIT_PORT + 3);
+        assertTrue(this.cluster.start(peer3.getEndpoint()));
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(100);
+        assertEquals(0, this.cluster.getFsmByPeer(peer3).getLogs().size());
+
+        assertTrue(this.cliService.addPeer(this.groupId, this.conf, peer3).isOk());
+        Thread.sleep(100);
+        assertEquals(10, this.cluster.getFsmByPeer(peer3).getLogs().size());
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(100);
+        assertEquals(6, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(20, fsm.getLogs().size());
+        }
+
+        //remove peer3
+        assertTrue(this.cliService.removePeer(this.groupId, this.conf, peer3).isOk());
+        Thread.sleep(200);
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        Thread.sleep(1000);
+        assertEquals(6, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            if (fsm.getAddress().equals(peer3.getEndpoint())) {
+                assertEquals(20, fsm.getLogs().size());
+            } else {
+                assertEquals(30, fsm.getLogs().size());
+            }
+        }
+    }
+
+    @Test
+    public void testChangePeers() throws Exception {
+        final List newPeers = TestUtils.generatePeers(10);
+        newPeers.removeAll(this.conf.getPeerSet());
+        for (final PeerId peer : newPeers) {
+            assertTrue(this.cluster.start(peer.getEndpoint()));
+        }
+        this.cluster.waitLeader();
+        final Node oldLeaderNode = this.cluster.getLeader();
+        assertNotNull(oldLeaderNode);
+        final PeerId oldLeader = oldLeaderNode.getNodeId().getPeerId();
+        assertNotNull(oldLeader);
+        assertTrue(this.cliService.changePeers(this.groupId, this.conf, new Configuration(newPeers)).isOk());
+        this.cluster.waitLeader();
+        final PeerId newLeader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotEquals(oldLeader, newLeader);
+        assertTrue(newPeers.contains(newLeader));
+    }
+
+    @Test
+    public void testSnapshot() throws Exception {
+        sendTestTaskAndWait(this.cluster.getLeader(), 0);
+        assertEquals(5, this.cluster.getFsms().size());
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(0, fsm.getSaveSnapshotTimes());
+        }
+
+        for (final PeerId peer : this.conf) {
+            assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+        }
+        for (final PeerId peer : this.conf.getLearners()) {
+            assertTrue(this.cliService.snapshot(this.groupId, peer).isOk());
+        }
+        Thread.sleep(1000);
+        for (final MockStateMachine fsm : this.cluster.getFsms()) {
+            assertEquals(1, fsm.getSaveSnapshotTimes());
+        }
+    }
+
+    @Test
+    public void testGetPeers() throws Exception {
+        PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+        // stop one peer
+        final List peers = this.conf.getPeers();
+        this.cluster.stop(peers.get(0).getEndpoint());
+
+        this.cluster.waitLeader();
+
+        leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getPeers(this.groupId, this.conf)).toArray());
+
+        this.cluster.stopAll();
+
+        try {
+            this.cliService.getPeers(this.groupId, this.conf);
+            fail();
+        } catch (final IllegalStateException e) {
+            assertTrue(e.getMessage().contains("Fail to get leader of group " + this.groupId));
+        }
+    }
+
+    @Test
+    public void testGetAlivePeers() throws Exception {
+        PeerId leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(this.conf.getPeerSet().toArray(),
+            new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+        // stop one peer
+        final List peers = this.conf.getPeers();
+        this.cluster.stop(peers.get(0).getEndpoint());
+        peers.remove(0);
+
+        this.cluster.waitLeader();
+
+        Thread.sleep(1000);
+
+        leader = this.cluster.getLeader().getNodeId().getPeerId();
+        assertNotNull(leader);
+        assertArrayEquals(new HashSet<>(peers).toArray(),
+            new HashSet<>(this.cliService.getAlivePeers(this.groupId, this.conf)).toArray());
+
+        this.cluster.stopAll();
+
+        try {
+            this.cliService.getAlivePeers(this.groupId, this.conf);
+            fail();
+        } catch (final IllegalStateException e) {
+            assertTrue(e.getMessage().contains("Fail to get leader of group " + this.groupId));
+            ;
+        }
+    }
+
+    @Test
+    public void testRebalance() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        groupIds.add("group_5");
+        groupIds.add("group_6");
+        groupIds.add("group_7");
+        groupIds.add("group_8");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockCliService(rebalancedLeaderIds, new PeerId("host_1", 8080));
+
+        assertTrue(cliService.rebalance(groupIds, conf, rebalancedLeaderIds).isOk());
+        assertEquals(groupIds.size(), rebalancedLeaderIds.size());
+
+        final Map ret = new HashMap<>();
+        for (Map.Entry entry : rebalancedLeaderIds.entrySet()) {
+            ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+        }
+        final int expectedAvgLeaderNum = (int) Math.ceil((double) groupIds.size() / conf.size());
+        for (Map.Entry entry : ret.entrySet()) {
+            System.out.println(entry);
+            assertTrue(entry.getValue() <= expectedAvgLeaderNum);
+        }
+    }
+
+    @Test
+    public void testRebalanceOnLeaderFail() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockLeaderFailCliService();
+
+        assertEquals("Fail to get leader", cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+    }
+
+    @Test
+    public void testRelalanceOnTransferLeaderFail() {
+        final Set groupIds = new TreeSet<>();
+        groupIds.add("group_1");
+        groupIds.add("group_2");
+        groupIds.add("group_3");
+        groupIds.add("group_4");
+        groupIds.add("group_5");
+        groupIds.add("group_6");
+        groupIds.add("group_7");
+        final Configuration conf = new Configuration();
+        conf.addPeer(new PeerId("host_1", 8080));
+        conf.addPeer(new PeerId("host_2", 8080));
+        conf.addPeer(new PeerId("host_3", 8080));
+
+        final Map rebalancedLeaderIds = new HashMap<>();
+
+        final CliService cliService = new MockTransferLeaderFailCliService(rebalancedLeaderIds,
+            new PeerId("host_1", 8080));
+
+        assertEquals("Fail to transfer leader",
+            cliService.rebalance(groupIds, conf, rebalancedLeaderIds).getErrorMsg());
+        assertTrue(groupIds.size() >= rebalancedLeaderIds.size());
+
+        final Map ret = new HashMap<>();
+        for (Map.Entry entry : rebalancedLeaderIds.entrySet()) {
+            ret.compute(entry.getValue(), (ignored, num) -> num == null ? 1 : num + 1);
+        }
+        for (Map.Entry entry : ret.entrySet()) {
+            System.out.println(entry);
+            assertEquals(new PeerId("host_1", 8080), entry.getKey());
+        }
+    }
+
+    class MockCliService extends CliServiceImpl {
+
+        private final Map rebalancedLeaderIds;
+        private final PeerId              initialLeaderId;
+
+        MockCliService(final Map rebalancedLeaderIds, final PeerId initialLeaderId) {
+            this.rebalancedLeaderIds = rebalancedLeaderIds;
+            this.initialLeaderId = initialLeaderId;
+        }
+
+        @Override
+        public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+            final PeerId ret = this.rebalancedLeaderIds.get(groupId);
+            if (ret != null) {
+                leaderId.parse(ret.toString());
+            } else {
+                leaderId.parse(this.initialLeaderId.toString());
+            }
+            return Status.OK();
+        }
+
+        @Override
+        public List getAlivePeers(final String groupId, final Configuration conf) {
+            return conf.getPeers();
+        }
+
+        @Override
+        public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+            return Status.OK();
+        }
+    }
+
+    class MockLeaderFailCliService extends MockCliService {
+
+        MockLeaderFailCliService() {
+            super(null, null);
+        }
+
+        @Override
+        public Status getLeader(final String groupId, final Configuration conf, final PeerId leaderId) {
+            return new Status(-1, "Fail to get leader");
+        }
+    }
+
+    class MockTransferLeaderFailCliService extends MockCliService {
+
+        MockTransferLeaderFailCliService(final Map rebalancedLeaderIds, final PeerId initialLeaderId) {
+            super(rebalancedLeaderIds, initialLeaderId);
+        }
+
+        @Override
+        public Status transferLeader(final String groupId, final Configuration conf, final PeerId peer) {
+            return new Status(-1, "Fail to transfer leader");
+        }
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
new file mode 100644
index 0000000..111151b
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/ExpectClosure.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.util.concurrent.CountDownLatch;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+
+import static org.junit.Assert.assertEquals;
+
+public class ExpectClosure implements Closure {
+    private int            expectedErrCode;
+    private String         expectErrMsg;
+    private CountDownLatch latch;
+
+    public ExpectClosure(CountDownLatch latch) {
+        this(RaftError.SUCCESS, latch);
+    }
+
+    public ExpectClosure(RaftError expectedErrCode, CountDownLatch latch) {
+        this(expectedErrCode, null, latch);
+
+    }
+
+    public ExpectClosure(RaftError expectedErrCode, String expectErrMsg, CountDownLatch latch) {
+        super();
+        this.expectedErrCode = expectedErrCode.getNumber();
+        this.expectErrMsg = expectErrMsg;
+        this.latch = latch;
+    }
+
+    public ExpectClosure(int code, String expectErrMsg, CountDownLatch latch) {
+        super();
+        this.expectedErrCode = code;
+        this.expectErrMsg = expectErrMsg;
+        this.latch = latch;
+    }
+
+    @Override
+    public void run(Status status) {
+        if (this.expectedErrCode >= 0) {
+            assertEquals(this.expectedErrCode, status.getCode());
+        }
+        if (this.expectErrMsg != null) {
+            assertEquals(this.expectErrMsg, status.getErrorMsg());
+        }
+        latch.countDown();
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
new file mode 100644
index 0000000..2b67748
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockClosure.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+
+class MockClosure implements Closure {
+    Status s;
+
+    @Override
+    public void run(Status status) {
+        this.s = status;
+
+    }
+}
\ No newline at end of file
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
new file mode 100644
index 0000000..0191396
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/MockStateMachine.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+public class MockStateMachine extends StateMachineAdapter {
+
+    private final Lock             lock                  = new ReentrantLock();
+    private volatile int           onStartFollowingTimes = 0;
+    private volatile int           onStopFollowingTimes  = 0;
+    private volatile long          leaderTerm            = -1;
+    private volatile long          appliedIndex          = -1;
+    private volatile long          snapshotIndex         = -1L;
+    private final List logs                  = new ArrayList<>();
+    private final Endpoint         address;
+    private volatile int           saveSnapshotTimes;
+    private volatile int           loadSnapshotTimes;
+
+    public Endpoint getAddress() {
+        return this.address;
+    }
+
+    public MockStateMachine(final Endpoint address) {
+        super();
+        this.address = address;
+    }
+
+    public int getSaveSnapshotTimes() {
+        return this.saveSnapshotTimes;
+    }
+
+    public int getLoadSnapshotTimes() {
+        return this.loadSnapshotTimes;
+    }
+
+    public int getOnStartFollowingTimes() {
+        return this.onStartFollowingTimes;
+    }
+
+    public int getOnStopFollowingTimes() {
+        return this.onStopFollowingTimes;
+    }
+
+    public long getLeaderTerm() {
+        return this.leaderTerm;
+    }
+
+    public long getAppliedIndex() {
+        return this.appliedIndex;
+    }
+
+    public long getSnapshotIndex() {
+        return this.snapshotIndex;
+    }
+
+    public void lock() {
+        this.lock.lock();
+    }
+
+    public void unlock() {
+        this.lock.unlock();
+    }
+
+    public List getLogs() {
+        this.lock.lock();
+        try {
+            return this.logs;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    private final AtomicLong lastAppliedIndex = new AtomicLong(-1);
+
+    @Override
+    public void onApply(final Iterator iter) {
+        while (iter.hasNext()) {
+            this.lock.lock();
+            try {
+                if (iter.getIndex() <= this.lastAppliedIndex.get()) {
+                    //prevent duplication
+                    continue;
+                }
+                this.lastAppliedIndex.set(iter.getIndex());
+                this.logs.add(iter.getData().slice());
+                if (iter.done() != null) {
+                    iter.done().run(Status.OK());
+                }
+            } finally {
+                this.lock.unlock();
+            }
+            this.appliedIndex = iter.getIndex();
+            iter.next();
+        }
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm > 0;
+    }
+
+    @Override
+    public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+        this.saveSnapshotTimes++;
+        final String path = writer.getPath() + File.separator + "data";
+        final File file = new File(path);
+        try (FileOutputStream fout = new FileOutputStream(file);
+                BufferedOutputStream out = new BufferedOutputStream(fout)) {
+            this.lock.lock();
+            try {
+                for (final ByteBuffer buf : this.logs) {
+                    final byte[] bs = new byte[4];
+                    Bits.putInt(bs, 0, buf.remaining());
+                    out.write(bs);
+                    out.write(buf.array());
+                }
+                this.snapshotIndex = this.appliedIndex;
+            } finally {
+                this.lock.unlock();
+            }
+            System.out.println("Node<" + this.address + "> saved snapshot into " + file);
+            writer.addFile("data");
+            done.run(Status.OK());
+        } catch (final IOException e) {
+            e.printStackTrace();
+            done.run(new Status(RaftError.EIO, "Fail to save snapshot"));
+        }
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        this.lastAppliedIndex.set(0);
+        this.loadSnapshotTimes++;
+        final String path = reader.getPath() + File.separator + "data";
+        final File file = new File(path);
+        if (!file.exists()) {
+            return false;
+        }
+        try (FileInputStream fin = new FileInputStream(file); BufferedInputStream in = new BufferedInputStream(fin)) {
+            this.lock.lock();
+            this.logs.clear();
+            try {
+                while (true) {
+                    final byte[] bs = new byte[4];
+                    if (in.read(bs) == 4) {
+                        final int len = Bits.getInt(bs, 0);
+                        final byte[] buf = new byte[len];
+                        if (in.read(buf) != len) {
+                            break;
+                        }
+                        this.logs.add(ByteBuffer.wrap(buf));
+                    } else {
+                        break;
+                    }
+                }
+            } finally {
+                this.lock.unlock();
+            }
+            System.out.println("Node<" + this.address + "> loaded snapshot from " + path);
+            return true;
+        } catch (final IOException e) {
+            e.printStackTrace();
+            return false;
+        }
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        super.onLeaderStart(term);
+        this.leaderTerm = term;
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        super.onLeaderStop(status);
+        this.leaderTerm = -1;
+    }
+
+    @Override
+    public void onStopFollowing(final LeaderChangeContext ctx) {
+        super.onStopFollowing(ctx);
+        this.onStopFollowingTimes++;
+    }
+
+    @Override
+    public void onStartFollowing(final LeaderChangeContext ctx) {
+        super.onStartFollowing(ctx);
+        this.onStartFollowingTimes++;
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
new file mode 100644
index 0000000..30d97ed
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/core/TestCluster.java
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.core;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.storage.SnapshotThrottle;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Test cluster for NodeTest
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-20 1:41:17 PM
+ */
+public class TestCluster {
+
+    static class Clusters {
+
+        public final IdentityHashMap needCloses = new IdentityHashMap<>();
+        private final Object                              EXIST      = new Object();
+
+        public synchronized void add(final TestCluster cluster) {
+            this.needCloses.put(cluster, EXIST);
+        }
+
+        public synchronized boolean remove(final TestCluster cluster) {
+            return this.needCloses.remove(cluster) != null;
+        }
+
+        public synchronized boolean isEmpty() {
+            return this.needCloses.isEmpty();
+        }
+
+        public synchronized List removeAll() {
+            final List clusters = new ArrayList<>(this.needCloses.keySet());
+            this.needCloses.clear();
+            return clusters;
+        }
+    }
+
+    public static final Clusters                          CLUSTERS  = new Clusters();
+
+    private final String                                  dataPath;
+    private final String                                  name;                                 // groupId
+    private final List                            peers;
+    private final List                          nodes;
+    private final LinkedHashMap fsms;
+    private final ConcurrentMap serverMap = new ConcurrentHashMap<>();
+    private final int                                     electionTimeoutMs;
+    private final Lock                                    lock      = new ReentrantLock();
+
+    private LinkedHashSet                         learners;
+
+    public LinkedHashSet getLearners() {
+        return this.learners;
+    }
+
+    public void setLearners(final LinkedHashSet learners) {
+        this.learners = learners;
+    }
+
+    public List getPeers() {
+        return this.peers;
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers) {
+        this(name, dataPath, peers, 300);
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers, final int electionTimeoutMs) {
+        this(name, dataPath, peers, new LinkedHashSet<>(), 300);
+    }
+
+    public TestCluster(final String name, final String dataPath, final List peers,
+                       final LinkedHashSet learners, final int electionTimeoutMs) {
+        super();
+        this.name = name;
+        this.dataPath = dataPath;
+        this.peers = peers;
+        this.nodes = new ArrayList<>(this.peers.size());
+        this.fsms = new LinkedHashMap<>(this.peers.size());
+        this.electionTimeoutMs = electionTimeoutMs;
+        this.learners = learners;
+        CLUSTERS.add(this);
+    }
+
+    public boolean start(final Endpoint addr) throws Exception {
+        return this.start(addr, false, 300);
+    }
+
+    public boolean start(final Endpoint addr, final int priority) throws Exception {
+        return this.start(addr, false, 300, false, null, null, priority);
+    }
+
+    public boolean startLearner(final PeerId peer) throws Exception {
+        this.learners.add(peer);
+        return this.start(peer.getEndpoint(), false, 300);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs)
+                                                                                                             throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, false);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics) throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, null, null);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle) throws IOException {
+        return this.start(listenAddr, emptyPeers, snapshotIntervalSecs, enableMetrics, snapshotThrottle, null);
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+                         final RaftOptions raftOptions, final int priority) throws IOException {
+
+        if (this.serverMap.get(listenAddr.toString()) != null) {
+            return true;
+        }
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+        nodeOptions.setEnableMetrics(enableMetrics);
+        nodeOptions.setSnapshotThrottle(snapshotThrottle);
+        nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+        if (raftOptions != null) {
+            nodeOptions.setRaftOptions(raftOptions);
+        }
+        final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        FileUtils.forceMkdir(new File(serverDataPath));
+        nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+        nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+        nodeOptions.setElectionPriority(priority);
+
+        final MockStateMachine fsm = new MockStateMachine(listenAddr);
+        nodeOptions.setFsm(fsm);
+
+        if (!emptyPeers) {
+            nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+        }
+
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+        final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0, priority),
+            nodeOptions, rpcServer);
+
+        this.lock.lock();
+        try {
+            if (this.serverMap.put(listenAddr.toString(), server) == null) {
+                final Node node = server.start();
+
+                this.fsms.put(new PeerId(listenAddr, 0), fsm);
+                this.nodes.add((NodeImpl) node);
+                return true;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return false;
+    }
+
+    public boolean start(final Endpoint listenAddr, final boolean emptyPeers, final int snapshotIntervalSecs,
+                         final boolean enableMetrics, final SnapshotThrottle snapshotThrottle,
+                         final RaftOptions raftOptions) throws IOException {
+
+        if (this.serverMap.get(listenAddr.toString()) != null) {
+            return true;
+        }
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        nodeOptions.setElectionTimeoutMs(this.electionTimeoutMs);
+        nodeOptions.setEnableMetrics(enableMetrics);
+        nodeOptions.setSnapshotThrottle(snapshotThrottle);
+        nodeOptions.setSnapshotIntervalSecs(snapshotIntervalSecs);
+        if (raftOptions != null) {
+            nodeOptions.setRaftOptions(raftOptions);
+        }
+        final String serverDataPath = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        FileUtils.forceMkdir(new File(serverDataPath));
+        nodeOptions.setLogUri(serverDataPath + File.separator + "logs");
+        nodeOptions.setRaftMetaUri(serverDataPath + File.separator + "meta");
+        nodeOptions.setSnapshotUri(serverDataPath + File.separator + "snapshot");
+        final MockStateMachine fsm = new MockStateMachine(listenAddr);
+        nodeOptions.setFsm(fsm);
+
+        if (!emptyPeers) {
+            nodeOptions.setInitialConf(new Configuration(this.peers, this.learners));
+        }
+
+        final RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(listenAddr);
+        final RaftGroupService server = new RaftGroupService(this.name, new PeerId(listenAddr, 0), nodeOptions,
+            rpcServer);
+
+        this.lock.lock();
+        try {
+            if (this.serverMap.put(listenAddr.toString(), server) == null) {
+                final Node node = server.start();
+
+                this.fsms.put(new PeerId(listenAddr, 0), fsm);
+                this.nodes.add((NodeImpl) node);
+                return true;
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return false;
+    }
+
+    public MockStateMachine getFsmByPeer(final PeerId peer) {
+        this.lock.lock();
+        try {
+            return this.fsms.get(peer);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public List getFsms() {
+        this.lock.lock();
+        try {
+            return new ArrayList<>(this.fsms.values());
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public boolean stop(final Endpoint listenAddr) throws InterruptedException {
+        final Node node = removeNode(listenAddr);
+        final CountDownLatch latch = new CountDownLatch(1);
+        if (node != null) {
+            node.shutdown(new ExpectClosure(latch));
+            node.join();
+            latch.await();
+        }
+        final RaftGroupService raftGroupService = this.serverMap.remove(listenAddr.toString());
+        raftGroupService.shutdown();
+        raftGroupService.join();
+        return node != null;
+    }
+
+    public void stopAll() throws InterruptedException {
+        final List addrs = getAllNodes();
+        final List nodes = new ArrayList<>();
+        for (final Endpoint addr : addrs) {
+            final Node node = removeNode(addr);
+            node.shutdown();
+            nodes.add(node);
+            this.serverMap.remove(addr.toString()).shutdown();
+        }
+        for (final Node node : nodes) {
+            node.join();
+        }
+        CLUSTERS.remove(this);
+    }
+
+    public void clean(final Endpoint listenAddr) throws IOException {
+        final String path = this.dataPath + File.separator + listenAddr.toString().replace(':', '_');
+        System.out.println("Clean dir:" + path);
+        FileUtils.deleteDirectory(new File(path));
+    }
+
+    public Node getLeader() {
+        this.lock.lock();
+        try {
+            for (int i = 0; i < this.nodes.size(); i++) {
+                final NodeImpl node = this.nodes.get(i);
+                if (node.isLeader() && this.fsms.get(node.getServerId()).getLeaderTerm() == node.getCurrentTerm()) {
+                    return node;
+                }
+            }
+            return null;
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public MockStateMachine getLeaderFsm() {
+        final Node leader = getLeader();
+        if (leader != null) {
+            return (MockStateMachine) leader.getOptions().getFsm();
+        }
+        return null;
+    }
+
+    public void waitLeader() throws InterruptedException {
+        while (true) {
+            final Node node = getLeader();
+            if (node != null) {
+                return;
+            } else {
+                Thread.sleep(10);
+            }
+        }
+    }
+
+    public List getFollowers() {
+        final List ret = new ArrayList<>();
+        this.lock.lock();
+        try {
+            for (final NodeImpl node : this.nodes) {
+                if (!node.isLeader() && !this.learners.contains(node.getServerId())) {
+                    ret.add(node);
+                }
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return ret;
+    }
+
+    /**
+     * Ensure all peers leader is expectAddr
+     * @param expectAddr expected address
+     * @throws InterruptedException if interrupted
+     */
+    public void ensureLeader(final Endpoint expectAddr) throws InterruptedException {
+        while (true) {
+            this.lock.lock();
+            for (final Node node : this.nodes) {
+                final PeerId leaderId = node.getLeaderId();
+                if (!leaderId.getEndpoint().equals(expectAddr)) {
+                    this.lock.unlock();
+                    Thread.sleep(10);
+                    continue;
+                }
+            }
+            // all is ready
+            this.lock.unlock();
+            return;
+        }
+    }
+
+    public List getNodes() {
+        this.lock.lock();
+        try {
+            return new ArrayList<>(this.nodes);
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public List getAllNodes() {
+        this.lock.lock();
+        try {
+            return this.nodes.stream().map(node -> node.getNodeId().getPeerId().getEndpoint())
+                .collect(Collectors.toList());
+        } finally {
+            this.lock.unlock();
+        }
+    }
+
+    public Node removeNode(final Endpoint addr) {
+        Node ret = null;
+        this.lock.lock();
+        try {
+            for (int i = 0; i < this.nodes.size(); i++) {
+                if (this.nodes.get(i).getNodeId().getPeerId().getEndpoint().equals(addr)) {
+                    ret = this.nodes.remove(i);
+                    this.fsms.remove(ret.getNodeId().getPeerId());
+                    break;
+                }
+            }
+        } finally {
+            this.lock.unlock();
+        }
+        return ret;
+    }
+
+    public boolean ensureSame() throws InterruptedException {
+        return this.ensureSame(-1);
+    }
+
+    /**
+     * Ensure all logs is the same in all nodes.
+     * @param waitTimes
+     * @return
+     * @throws InterruptedException
+     */
+    public boolean ensureSame(final int waitTimes) throws InterruptedException {
+        this.lock.lock();
+        List fsmList = new ArrayList<>(this.fsms.values());
+        if (fsmList.size() <= 1) {
+            this.lock.unlock();
+            return true;
+        }
+        System.out.println("Start ensureSame, waitTimes=" + waitTimes);
+        try {
+            int nround = 0;
+            final MockStateMachine first = fsmList.get(0);
+            CHECK: while (true) {
+                first.lock();
+                if (first.getLogs().isEmpty()) {
+                    first.unlock();
+                    Thread.sleep(10);
+                    nround++;
+                    if (waitTimes > 0 && nround > waitTimes) {
+                        return false;
+                    }
+                    continue CHECK;
+                }
+
+                for (int i = 1; i < fsmList.size(); i++) {
+                    final MockStateMachine fsm = fsmList.get(i);
+                    fsm.lock();
+                    if (fsm.getLogs().size() != first.getLogs().size()) {
+                        fsm.unlock();
+                        first.unlock();
+                        Thread.sleep(10);
+                        nround++;
+                        if (waitTimes > 0 && nround > waitTimes) {
+                            return false;
+                        }
+                        continue CHECK;
+                    }
+
+                    for (int j = 0; j < first.getLogs().size(); j++) {
+                        final ByteBuffer firstData = first.getLogs().get(j);
+                        final ByteBuffer fsmData = fsm.getLogs().get(j);
+                        if (!firstData.equals(fsmData)) {
+                            fsm.unlock();
+                            first.unlock();
+                            Thread.sleep(10);
+                            nround++;
+                            if (waitTimes > 0 && nround > waitTimes) {
+                                return false;
+                            }
+                            continue CHECK;
+                        }
+                    }
+                    fsm.unlock();
+                }
+                first.unlock();
+                break;
+            }
+            return true;
+        } finally {
+            this.lock.unlock();
+            System.out.println("End ensureSame, waitTimes=" + waitTimes);
+        }
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
new file mode 100644
index 0000000..bdb1409
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/AbstractClientServiceTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.InvokeTimeoutException;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RemotingException;
+import com.alipay.sofa.jraft.option.RpcOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.rpc.impl.AbstractClientService;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.eq;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class AbstractClientServiceTest {
+    static class MockClientService extends AbstractClientService {
+        public void setRpcClient(final RpcClient rpcClient) {
+            this.rpcClient = rpcClient;
+        }
+    }
+
+    private RpcOptions         rpcOptions;
+    private MockClientService  clientService;
+    @Mock
+    private RpcClient          rpcClient;
+    private RpcResponseFactory rpcResponseFactory = RpcFactoryHelper.responseFactory();
+    private final Endpoint     endpoint           = new Endpoint("localhost", 8081);
+
+    @Before
+    public void setup() {
+        this.rpcOptions = new RpcOptions();
+        this.clientService = new MockClientService();
+        assertTrue(this.clientService.init(this.rpcOptions));
+        this.clientService.setRpcClient(this.rpcClient);
+
+    }
+
+    @Test
+    public void testConnect() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenReturn(this.rpcResponseFactory.newResponse(null, Status.OK()));
+        assertTrue(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testConnectFailure() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenReturn(this.rpcResponseFactory.newResponse(null, new Status(-1, "test")));
+        assertFalse(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testConnectException() throws Exception {
+        Mockito.when(
+            this.rpcClient.invokeSync(eq(this.endpoint), Mockito.any(),
+                eq((long) this.rpcOptions.getRpcConnectTimeoutMs()))) //
+            .thenThrow(new RemotingException("test"));
+        assertFalse(this.clientService.connect(this.endpoint));
+    }
+
+    @Test
+    public void testDisconnect() {
+        this.clientService.disconnect(this.endpoint);
+        Mockito.verify(this.rpcClient).closeConnection(this.endpoint);
+    }
+
+    static class MockRpcResponseClosure extends RpcResponseClosureAdapter {
+
+        CountDownLatch latch = new CountDownLatch(1);
+
+        Status         status;
+
+        @Override
+        public void run(final Status status) {
+            this.status = status;
+            this.latch.countDown();
+        }
+
+    }
+
+    @Test
+    public void testCancel() throws Exception {
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        future.cancel(true);
+        ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+        cb.complete(response, null);
+
+        // The closure should be notified with ECANCELED error code.
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.ECANCELED.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneOK() throws Exception {
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), Mockito.any(),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        ErrorResponse response = (ErrorResponse) this.rpcResponseFactory.newResponse(null, Status.OK());
+        cb.complete(response, null);
+
+        Message msg = future.get();
+        assertNotNull(msg);
+        assertTrue(msg instanceof ErrorResponse);
+        assertSame(msg, response);
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(0, done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneException() throws Exception {
+        InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        Mockito
+            .doThrow(new RemotingException())
+            .when(this.rpcClient)
+            .invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx), callbackArg.capture(),
+                eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertTrue(future.isDone());
+
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof RemotingException);
+        }
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.EINTERNAL.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDoneOnException() throws Exception {
+        InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        PingRequest request = TestUtils.createPingRequest();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        cb.complete(null, new InvokeTimeoutException());
+
+        try {
+            future.get();
+            fail();
+        } catch (ExecutionException e) {
+            assertTrue(e.getCause() instanceof InvokeTimeoutException);
+        }
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertEquals(RaftError.ETIMEDOUT.getNumber(), done.status.getCode());
+    }
+
+    @Test
+    public void testInvokeWithDOneOnErrorResponse() throws Exception {
+        final InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(InvokeContext.CRC_SWITCH, false);
+        final ArgumentCaptor callbackArg = ArgumentCaptor.forClass(InvokeCallback.class);
+        final CliRequests.GetPeersRequest request = CliRequests.GetPeersRequest.newBuilder() //
+            .setGroupId("id") //
+            .setLeaderId("127.0.0.1:8001") //
+            .build();
+
+        MockRpcResponseClosure done = new MockRpcResponseClosure<>();
+        Future future = this.clientService.invokeWithDone(this.endpoint, request, invokeCtx, done, -1);
+        Mockito.verify(this.rpcClient).invokeAsync(eq(this.endpoint), eq(request), eq(invokeCtx),
+            callbackArg.capture(), eq((long) this.rpcOptions.getRpcDefaultTimeout()));
+        InvokeCallback cb = callbackArg.getValue();
+        assertNotNull(cb);
+        assertNotNull(future);
+
+        assertNull(done.getResponse());
+        assertNull(done.status);
+        assertFalse(future.isDone());
+
+        final Message resp = this.rpcResponseFactory.newResponse(CliRequests.GetPeersResponse.getDefaultInstance(),
+            new Status(-1, "failed"));
+        cb.complete(resp, null);
+
+        final Message msg = future.get();
+
+        assertTrue(msg instanceof ErrorResponse);
+        assertEquals(((ErrorResponse) msg).getErrorMsg(), "failed");
+
+        done.latch.await();
+        assertNotNull(done.status);
+        assertTrue(!done.status.isOk());
+        assertEquals(done.status.getErrorMsg(), "failed");
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java
new file mode 100644
index 0000000..92cf3fe
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/ConnectionRefreshTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rpc.impl.PingRequestProcessor;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ConnectionRefreshTest {
+
+    @Ignore
+    @Test
+    public void simulation() throws InterruptedException {
+        ProtobufMsgFactory.load();
+
+        final RpcServer server = RpcFactoryHelper.rpcFactory().createRpcServer(new Endpoint("127.0.0.1", 19991));
+        server.registerProcessor(new PingRequestProcessor());
+        server.init(null);
+
+        final Endpoint target = new Endpoint("my.test.host1.com", 19991);
+
+        final RpcClient client = RpcFactoryHelper.rpcFactory().createRpcClient();
+        client.init(null);
+
+        final RpcRequests.PingRequest req = RpcRequests.PingRequest.newBuilder() //
+            .setSendTimestamp(System.currentTimeMillis()) //
+            .build();
+
+        for (int i = 0; i < 1000; i++) {
+            try {
+                final Object resp = client.invokeSync(target, req, 3000);
+                System.out.println(resp);
+            } catch (final Exception e) {
+                e.printStackTrace();
+            }
+            Thread.sleep(1000);
+        }
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
new file mode 100644
index 0000000..e7d137e
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/PingRequestProcessorTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+
+import static org.junit.Assert.assertEquals;
+
+public class PingRequestProcessorTest {
+
+    @Test
+    public void testHandlePing() throws Exception {
+        PingRequestProcessor processor = new PingRequestProcessor();
+        MockAsyncContext ctx = new MockAsyncContext();
+        processor.handleRequest(ctx, TestUtils.createPingRequest());
+        ErrorResponse response = (ErrorResponse) ctx.getResponseObject();
+        assertEquals(0, response.getErrorCode());
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java
new file mode 100644
index 0000000..9d0220a
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/AppendEntriesRequestProcessorTest.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import org.junit.Test;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcRequests.AppendEntriesRequest;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.rpc.impl.core.AppendEntriesRequestProcessor.PeerPair;
+import com.alipay.sofa.jraft.rpc.impl.core.AppendEntriesRequestProcessor.PeerRequestContext;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.concurrent.ConcurrentHashSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.mockito.Matchers.eq;
+
+public class AppendEntriesRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private AppendEntriesRequest request;
+
+    private final String         serverId = "localhost:8082";
+
+    @Override
+    public AppendEntriesRequest createRequest(final String groupId, final PeerId peerId) {
+        this.request = AppendEntriesRequest.newBuilder().setCommittedIndex(0). //
+            setGroupId(groupId). //
+            setPeerId(peerId.toString()).//
+            setServerId(this.serverId). //
+            setPrevLogIndex(0). //
+            setTerm(0). //
+            setPrevLogTerm(0).build();
+        return this.request;
+    }
+
+    @Mock
+    private Connection conn;
+
+    @Override
+    public void setup() {
+        super.setup();
+        this.asyncContext = new MockAsyncContext() {
+            @Override
+            public Connection getConnection() {
+                return AppendEntriesRequestProcessorTest.this.conn;
+            }
+        };
+        Set pairs = new ConcurrentHashSet<>();
+        pairs.add(new PeerPair(this.peerIdStr, this.serverId));
+        Mockito.when(this.conn.getAttribute(AppendEntriesRequestProcessor.PAIR_ATTR)).thenReturn(pairs);
+    }
+
+    private ExecutorService executor;
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        this.executor = Executors.newSingleThreadExecutor();
+        return new AppendEntriesRequestProcessor(this.executor);
+    }
+
+    @Override
+    public void teardown() {
+        super.teardown();
+        if (this.executor != null) {
+            this.executor.shutdownNow();
+        }
+    }
+
+    @Test
+    public void testPairOf() {
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+
+        PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        assertEquals(pair.remote, this.serverId);
+        assertEquals(pair.local, this.peerIdStr);
+
+        // test constant pool
+        assertSame(pair, processor.pairOf(this.peerIdStr, this.serverId));
+        assertSame(pair, processor.pairOf(this.peerIdStr, this.serverId));
+    }
+
+    @Test
+    public void testOnClosed() {
+        mockNode();
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+
+        PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+        assertSame(ctx, processor.getPeerRequestContext(this.groupId, pair));
+        assertSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+
+        processor.onClosed(null, this.conn);
+        assertNull(processor.getPeerRequestContext(this.groupId, pair));
+        assertNotSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+    }
+
+    @Override
+    public void verify(final String interest, final RaftServerService service,
+                       final NodeRequestProcessor processor) {
+        assertEquals(interest, AppendEntriesRequest.class.getName());
+        Mockito.verify(service).handleAppendEntriesRequest(eq(this.request), Mockito.any());
+        final PeerPair pair = ((AppendEntriesRequestProcessor) processor).pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = ((AppendEntriesRequestProcessor) processor).getOrCreatePeerRequestContext(
+            this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+    }
+
+    @Test
+    public void testGetPeerRequestContextRemovePeerRequestContext() {
+        mockNode();
+
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(ctx);
+        assertSame(ctx, processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn));
+        assertEquals(0, ctx.getNextRequiredSequence());
+        assertEquals(0, ctx.getAndIncrementSequence());
+        assertEquals(1, ctx.getAndIncrementSequence());
+        assertEquals(0, ctx.getAndIncrementNextRequiredSequence());
+        assertEquals(1, ctx.getAndIncrementNextRequiredSequence());
+        assertFalse(ctx.hasTooManyPendingResponses());
+
+        processor.removePeerRequestContext(this.groupId, pair);
+        final PeerRequestContext newCtx = processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        assertNotNull(newCtx);
+        assertNotSame(ctx, newCtx);
+
+        assertEquals(0, newCtx.getNextRequiredSequence());
+        assertEquals(0, newCtx.getAndIncrementSequence());
+        assertEquals(1, newCtx.getAndIncrementSequence());
+        assertEquals(0, newCtx.getAndIncrementNextRequiredSequence());
+        assertEquals(1, newCtx.getAndIncrementNextRequiredSequence());
+        assertFalse(newCtx.hasTooManyPendingResponses());
+    }
+
+    @Test
+    public void testSendSequenceResponse() {
+        mockNode();
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        processor.getOrCreatePeerRequestContext(this.groupId, pair, this.conn);
+        final PingRequest msg = TestUtils.createPingRequest();
+        final RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        processor.sendSequenceResponse(this.groupId, pair, 1, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.never()).sendResponse(msg);
+
+        processor.sendSequenceResponse(this.groupId, pair, 0, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.times(2)).sendResponse(msg);
+    }
+
+    @Test
+    public void testTooManyPendingResponses() {
+        final PeerId peer = mockNode();
+        NodeManager.getInstance().get(this.groupId, peer).getRaftOptions().setMaxReplicatorInflightMsgs(2);
+
+        final RpcContext asyncContext = Mockito.mock(RpcContext.class);
+        final AppendEntriesRequestProcessor processor = (AppendEntriesRequestProcessor) newProcessor();
+        final PeerPair pair = processor.pairOf(this.peerIdStr, this.serverId);
+        final PingRequest msg = TestUtils.createPingRequest();
+        final Connection conn = Mockito.mock(Connection.class);
+        Mockito.when(asyncContext.getConnection()).thenReturn(conn);
+        final PeerRequestContext ctx = processor.getOrCreatePeerRequestContext(this.groupId, pair, conn);
+        assertNotNull(ctx);
+        processor.sendSequenceResponse(this.groupId, pair, 1, asyncContext, msg);
+        processor.sendSequenceResponse(this.groupId, pair, 2, asyncContext, msg);
+        processor.sendSequenceResponse(this.groupId, pair, 3, asyncContext, msg);
+        Mockito.verify(asyncContext, Mockito.never()).sendResponse(msg);
+        Mockito.verify(conn).close();
+
+        final PeerRequestContext newCtx = processor.getOrCreatePeerRequestContext(this.groupId, pair, conn);
+        assertNotNull(newCtx);
+        assertNotSame(ctx, newCtx);
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java
new file mode 100644
index 0000000..c0161e7
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/BaseNodeRequestProcessorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.RaftOptions;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.google.protobuf.Message;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public abstract class BaseNodeRequestProcessorTest {
+    @Mock(extraInterfaces = { RaftServerService.class })
+    private Node               node;
+    protected final String     groupId   = "test";
+    protected final String     peerIdStr = "localhost:8081";
+    protected MockAsyncContext asyncContext;
+
+    public abstract T createRequest(String groupId, PeerId peerId);
+
+    public abstract NodeRequestProcessor newProcessor();
+
+    public abstract void verify(String interest, RaftServerService service, NodeRequestProcessor processor);
+
+    @Before
+    public void setup() {
+        Mockito.when(node.getRaftOptions()).thenReturn(new RaftOptions());
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testHandleRequest() {
+        final PeerId peerId = mockNode();
+
+        final NodeRequestProcessor processor = newProcessor();
+        processor.handleRequest(asyncContext, createRequest(groupId, peerId));
+        verify(processor.interest(), (RaftServerService) this.node, processor);
+    }
+
+    protected PeerId mockNode() {
+        Mockito.when(node.getGroupId()).thenReturn(this.groupId);
+        final PeerId peerId = new PeerId();
+        peerId.parse(this.peerIdStr);
+        Mockito.when(node.getNodeId()).thenReturn(new NodeId(groupId, peerId));
+        NodeManager.getInstance().addAddress(peerId.getEndpoint());
+        NodeManager.getInstance().add(node);
+        return peerId;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java
new file mode 100644
index 0000000..b788cc9
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/DefaultRaftClientServiceTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import com.alipay.sofa.jraft.ReplicatorGroup;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+@RunWith(value = MockitoJUnitRunner.class)
+public class DefaultRaftClientServiceTest {
+    private DefaultRaftClientService clientService;
+    @Mock
+    private ReplicatorGroup          rgGroup;
+
+    private final Endpoint           endpoint = new Endpoint("localhost", 8081);
+
+    @Before
+    public void setup() {
+        this.clientService = new DefaultRaftClientService(this.rgGroup);
+        this.clientService.init(new NodeOptions());
+    }
+
+    @Test
+    public void testPreVote() {
+        this.clientService.preVote(this.endpoint, RequestVoteRequest.newBuilder(). //
+            setGroupId("test"). //
+            setLastLogIndex(1). //
+            setLastLogTerm(1). //
+            setPeerId("localhost:1010"). //
+            setTerm(1).setServerId("localhost:1011").setPreVote(true).build(), null);
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java
new file mode 100644
index 0000000..28a7259
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/InstallSnapshotRequestProcessorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.RaftOutter.SnapshotMeta;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.InstallSnapshotRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class InstallSnapshotRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private InstallSnapshotRequest request;
+
+    @Override
+    public InstallSnapshotRequest createRequest(String groupId, PeerId peerId) {
+        request = InstallSnapshotRequest.newBuilder().setGroupId(groupId)
+            . //
+            setServerId("localhostL8082")
+            . //
+            setPeerId(peerId.toString())
+            . //
+            setTerm(0)
+            . //
+            setMeta(SnapshotMeta.newBuilder().setLastIncludedIndex(1).setLastIncludedTerm(1).build()).setUri("test")
+            .build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new InstallSnapshotRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service,
+                       NodeRequestProcessor processor) {
+        assertEquals(interest, InstallSnapshotRequest.class.getName());
+        Mockito.verify(service).handleInstallSnapshot(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java
new file mode 100644
index 0000000..38519ee
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/NodeRequestProcessorTest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.NodeManager;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.NodeId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequestClosure;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ErrorResponse;
+import com.alipay.sofa.jraft.rpc.RpcRequests.PingRequest;
+import com.alipay.sofa.jraft.test.MockAsyncContext;
+import com.alipay.sofa.jraft.test.TestUtils;
+import com.alipay.sofa.jraft.util.RpcFactoryHelper;
+import com.google.protobuf.Message;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Mockito.withSettings;
+
+public class NodeRequestProcessorTest {
+
+    private static class MockRequestProcessor extends NodeRequestProcessor {
+
+        private String peerId;
+        private String groupId;
+
+        public MockRequestProcessor(String peerId, String groupId) {
+            super(null, null);
+            this.peerId = peerId;
+            this.groupId = groupId;
+        }
+
+        @Override
+        protected String getPeerId(PingRequest request) {
+            return this.peerId;
+        }
+
+        @Override
+        protected String getGroupId(PingRequest request) {
+            return this.groupId;
+        }
+
+        @Override
+        protected Message processRequest0(RaftServerService serviceService, PingRequest request, RpcRequestClosure done) {
+            return RpcFactoryHelper.responseFactory().newResponse(null, Status.OK());
+        }
+
+        @Override
+        public String interest() {
+            return PingRequest.class.getName();
+        }
+
+    }
+
+    private MockRequestProcessor processor;
+    private MockAsyncContext     asyncContext;
+
+    @Before
+    public void setup() {
+        this.asyncContext = new MockAsyncContext();
+        this.processor = new MockRequestProcessor("localhost:8081", "test");
+    }
+
+    @After
+    public void teardown() {
+        NodeManager.getInstance().clear();
+    }
+
+    @Test
+    public void testOK() {
+        Node node = Mockito.mock(Node.class, withSettings().extraInterfaces(RaftServerService.class));
+        Mockito.when(node.getGroupId()).thenReturn("test");
+        PeerId peerId = new PeerId("localhost", 8081);
+        Mockito.when(node.getNodeId()).thenReturn(new NodeId("test", peerId));
+        NodeManager.getInstance().addAddress(peerId.getEndpoint());
+        NodeManager.getInstance().add(node);
+
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(0, resp.getErrorCode());
+    }
+
+    @Test
+    public void testInvalidPeerId() {
+        this.processor = new MockRequestProcessor("localhost", "test");
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.EINVAL.getNumber(), resp.getErrorCode());
+        assertEquals("Fail to parse peerId: localhost", resp.getErrorMsg());
+    }
+
+    @Test
+    public void testPeerIdNotFound() {
+        this.processor.handleRequest(asyncContext, TestUtils.createPingRequest());
+        ErrorResponse resp = (ErrorResponse) asyncContext.getResponseObject();
+        assertNotNull(resp);
+        assertEquals(RaftError.ENOENT.getNumber(), resp.getErrorCode());
+        assertEquals("Peer id not found: localhost:8081, group: test", resp.getErrorMsg());
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java
new file mode 100644
index 0000000..148adc5
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/PreVoteRequestProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class PreVoteRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private RequestVoteRequest request;
+
+    @Override
+    public RequestVoteRequest createRequest(String groupId, PeerId peerId) {
+        request = RequestVoteRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0). //
+            setLastLogIndex(0). //
+            setLastLogTerm(0). //
+            setPreVote(false).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new RequestVoteRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, RequestVoteRequest.class.getName());
+        Mockito.verify(service).handleRequestVoteRequest(eq(request));
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java
new file mode 100644
index 0000000..9c52f45
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/ReadIndexRequestProcessorTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.ReadIndexRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class ReadIndexRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private ReadIndexRequest request;
+
+    @Override
+    public ReadIndexRequest createRequest(String groupId, PeerId peerId) {
+        request = ReadIndexRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new ReadIndexRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, ReadIndexRequest.class.getName());
+        Mockito.verify(service).handleReadIndexRequest(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java
new file mode 100644
index 0000000..a7580b9
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/RequestVoteRequestProcessorTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.RequestVoteRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class RequestVoteRequestProcessorTest extends BaseNodeRequestProcessorTest {
+    private RequestVoteRequest request;
+
+    @Override
+    public RequestVoteRequest createRequest(String groupId, PeerId peerId) {
+        request = RequestVoteRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0). //
+            setLastLogIndex(0). //
+            setLastLogTerm(0). //
+            setPreVote(true).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new RequestVoteRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, RequestVoteRequest.class.getName());
+        Mockito.verify(service).handlePreVoteRequest(eq(request));
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java
new file mode 100644
index 0000000..daa058f
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/rpc/impl/core/TimeoutNowRequestProcessorTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rpc.impl.core;
+
+import org.mockito.Mockito;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftServerService;
+import com.alipay.sofa.jraft.rpc.RpcRequests.TimeoutNowRequest;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.eq;
+
+public class TimeoutNowRequestProcessorTest extends BaseNodeRequestProcessorTest {
+
+    private TimeoutNowRequest request;
+
+    @Override
+    public TimeoutNowRequest createRequest(String groupId, PeerId peerId) {
+        request = TimeoutNowRequest.newBuilder().setGroupId(groupId). //
+            setServerId("localhostL8082"). //
+            setPeerId(peerId.toString()). //
+            setTerm(0).build();
+        return request;
+    }
+
+    @Override
+    public NodeRequestProcessor newProcessor() {
+        return new TimeoutNowRequestProcessor(null);
+    }
+
+    @Override
+    public void verify(String interest, RaftServerService service, NodeRequestProcessor processor) {
+        assertEquals(interest, TimeoutNowRequest.class.getName());
+        Mockito.verify(service).handleTimeoutNowRequest(eq(request), Mockito.any());
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java
new file mode 100644
index 0000000..0f07dae
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/MockAsyncContext.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test;
+
+import com.alipay.sofa.jraft.rpc.Connection;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.google.protobuf.Message;
+
+/**
+ * mock alipay remoting async context
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-19 6:05:46 PM
+ */
+public class MockAsyncContext implements RpcContext {
+    private Object responseObject;
+
+    public Object getResponseObject() {
+        return this.responseObject;
+    }
+
+    @SuppressWarnings("unchecked")
+    public  T as(Class t) {
+        return (T) this.responseObject;
+    }
+
+    public void setResponseObject(Object responseObject) {
+        this.responseObject = responseObject;
+    }
+
+    @Override
+    public void sendResponse(Object responseObject) {
+        this.responseObject = responseObject;
+
+    }
+
+    @Override
+    public Connection getConnection() {
+        return null;
+    }
+
+    @Override
+    public String getRemoteAddress() {
+        return "localhost:12345";
+    }
+
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java
new file mode 100644
index 0000000..7d8757c
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/java/com/alipay/sofa/jraft/test/TestUtils.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test;
+
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.conf.ConfigurationEntry;
+import com.alipay.sofa.jraft.entity.EnumOutter;
+import com.alipay.sofa.jraft.entity.LogEntry;
+import com.alipay.sofa.jraft.entity.LogId;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RpcRequests;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Test helper
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-11 10:16:07 AM
+ */
+public class TestUtils {
+
+    public static ConfigurationEntry getConfEntry(final String confStr, final String oldConfStr) {
+        ConfigurationEntry entry = new ConfigurationEntry();
+        entry.setConf(JRaftUtils.getConfiguration(confStr));
+        entry.setOldConf(JRaftUtils.getConfiguration(oldConfStr));
+        return entry;
+    }
+
+    public static String mkTempDir() {
+        return Paths.get(System.getProperty("java.io.tmpdir", "/tmp"), "jraft_test_" + System.nanoTime()).toString();
+    }
+
+    public static LogEntry mockEntry(final int index, final int term) {
+        return mockEntry(index, term, 0);
+    }
+
+    public static LogEntry mockEntry(final int index, final int term, final int dataSize) {
+        LogEntry entry = new LogEntry(EnumOutter.EntryType.ENTRY_TYPE_NO_OP);
+        entry.setId(new LogId(index, term));
+        if (dataSize > 0) {
+            byte[] bs = new byte[dataSize];
+            ThreadLocalRandom.current().nextBytes(bs);
+            entry.setData(ByteBuffer.wrap(bs));
+        }
+        return entry;
+    }
+
+    public static List mockEntries() {
+        return mockEntries(10);
+    }
+
+    public static String getMyIp() {
+        String ip = null;
+        try {
+            Enumeration interfaces = NetworkInterface.getNetworkInterfaces();
+            while (interfaces.hasMoreElements()) {
+                NetworkInterface iface = interfaces.nextElement();
+                // filters out 127.0.0.1 and inactive interfaces
+                if (iface.isLoopback() || !iface.isUp()) {
+                    continue;
+                }
+
+                Enumeration addresses = iface.getInetAddresses();
+                while (addresses.hasMoreElements()) {
+                    InetAddress addr = addresses.nextElement();
+                    if (addr instanceof Inet4Address) {
+                        ip = addr.getHostAddress();
+                        break;
+                    }
+                }
+            }
+            return ip;
+        } catch (SocketException e) {
+            return "localhost";
+        }
+    }
+
+    public static List mockEntries(final int n) {
+        List entries = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            LogEntry entry = mockEntry(i, i);
+            if (i > 0) {
+                entry.setData(ByteBuffer.wrap(String.valueOf(i).getBytes()));
+            }
+            entries.add(entry);
+        }
+        return entries;
+    }
+
+    public static RpcRequests.PingRequest createPingRequest() {
+        RpcRequests.PingRequest reqObject = RpcRequests.PingRequest.newBuilder()
+            .setSendTimestamp(System.currentTimeMillis()).build();
+        return reqObject;
+    }
+
+    public static final int INIT_PORT = 5003;
+
+    public static List generatePeers(final int n) {
+        List ret = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            ret.add(new PeerId(getMyIp(), INIT_PORT + i));
+        }
+        return ret;
+    }
+
+    public static List generatePriorityPeers(final int n, List priorities) {
+        List ret = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            Endpoint endpoint = new Endpoint(getMyIp(), INIT_PORT + i);
+            PeerId peerId = new PeerId(endpoint, 0, priorities.get(i));
+            ret.add(peerId);
+        }
+        return ret;
+    }
+
+    public static byte[] getRandomBytes() {
+        final byte[] requestContext = new byte[ThreadLocalRandom.current().nextInt(10) + 1];
+        ThreadLocalRandom.current().nextBytes(requestContext);
+        return requestContext;
+    }
+}
diff --git a/jraft-extension/rpc-grpc-impl/src/test/resources/log4j2.xml b/jraft-extension/rpc-grpc-impl/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..eaa5a75
--- /dev/null
+++ b/jraft-extension/rpc-grpc-impl/src/test/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-rheakv/pom.xml b/jraft-rheakv/pom.xml
new file mode 100644
index 0000000..437baaf
--- /dev/null
+++ b/jraft-rheakv/pom.xml
@@ -0,0 +1,17 @@
+
+
+    4.0.0
+    
+        jraft-parent
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+
+    jraft-rheakv
+    pom
+    jraft-rheakv ${project.version}
+    
+        rheakv-core
+        rheakv-pd
+    
+
diff --git a/jraft-rheakv/rheakv-core/pom.xml b/jraft-rheakv/rheakv-core/pom.xml
new file mode 100644
index 0000000..3c1d58c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/pom.xml
@@ -0,0 +1,98 @@
+
+
+    4.0.0
+    
+        jraft-rheakv
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+
+    jraft-rheakv-core
+    rheakv-core ${project.version}
+
+    
+        2.10.5.1
+        2.10.4
+    
+
+    
+        
+            ${project.groupId}
+            jraft-core
+        
+        
+            org.slf4j
+            slf4j-api
+        
+        
+            net.openhft
+            affinity
+        
+        
+            com.lmax
+            disruptor
+        
+        
+            commons-io
+            commons-io
+        
+        
+            org.apache.commons
+            commons-compress
+        
+        
+            com.alipay.sofa
+            bolt
+        
+        
+            org.rocksdb
+            rocksdbjni
+        
+        
+            io.protostuff
+            protostuff-core
+        
+        
+            io.protostuff
+            protostuff-runtime
+        
+        
+            com.fasterxml.jackson.dataformat
+            jackson-dataformat-yaml
+            ${jackson.dataformat.version}
+            test
+        
+        
+            com.fasterxml.jackson.core
+            jackson-databind
+            ${jackson.databind.version}
+            test
+        
+        
+        
+        
+        
+        
+        
+            junit
+            junit
+        
+        
+            org.hamcrest
+            hamcrest-library
+        
+        
+            org.mockito
+            mockito-all
+        
+        
+        
+            org.openjdk.jmh
+            jmh-core
+        
+        
+            org.openjdk.jmh
+            jmh-generator-annprocess
+        
+    
+
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DefaultRegionKVService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DefaultRegionKVService.java
new file mode 100644
index 0000000..8bf661b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DefaultRegionKVService.java
@@ -0,0 +1,702 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanResponse;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.NodeExecutor;
+import com.alipay.sofa.jraft.rhea.storage.RawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.KVParameterRequires;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * Rhea KV region RPC request processing service.
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultRegionKVService implements RegionKVService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultRegionKVService.class);
+
+    private final RegionEngine  regionEngine;
+    private final RawKVStore    rawKVStore;
+
+    public DefaultRegionKVService(RegionEngine regionEngine) {
+        this.regionEngine = regionEngine;
+        this.rawKVStore = regionEngine.getMetricsRawKVStore();
+    }
+
+    @Override
+    public long getRegionId() {
+        return this.regionEngine.getRegion().getId();
+    }
+
+    @Override
+    public RegionEpoch getRegionEpoch() {
+        return this.regionEngine.getRegion().getRegionEpoch();
+    }
+
+    @Override
+    public void handlePutRequest(final PutRequest request,
+                                 final RequestProcessClosure> closure) {
+        final PutResponse response = new PutResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "put.key");
+            final byte[] value = KVParameterRequires.requireNonNull(request.getValue(), "put.value");
+            this.rawKVStore.put(key, value, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleBatchPutRequest(final BatchPutRequest request,
+                                      final RequestProcessClosure> closure) {
+        final BatchPutResponse response = new BatchPutResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final List kvEntries = KVParameterRequires
+                .requireNonEmpty(request.getKvEntries(), "put.kvEntries");
+            this.rawKVStore.put(kvEntries, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handlePutIfAbsentRequest(final PutIfAbsentRequest request,
+                                         final RequestProcessClosure> closure) {
+        final PutIfAbsentResponse response = new PutIfAbsentResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "putIfAbsent.key");
+            final byte[] value = KVParameterRequires.requireNonNull(request.getValue(), "putIfAbsent.value");
+            this.rawKVStore.putIfAbsent(key, value, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((byte[]) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleGetAndPutRequest(final GetAndPutRequest request,
+                                       final RequestProcessClosure> closure) {
+        final GetAndPutResponse response = new GetAndPutResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "getAndPut.key");
+            final byte[] value = KVParameterRequires.requireNonNull(request.getValue(), "getAndPut.value");
+            this.rawKVStore.getAndPut(key, value, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((byte[]) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleCompareAndPutRequest(final CompareAndPutRequest request,
+                                           final RequestProcessClosure> closure) {
+        final CompareAndPutResponse response = new CompareAndPutResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "compareAndPut.key");
+            final byte[] expect = KVParameterRequires.requireNonNull(request.getExpect(), "compareAndPut.expect");
+            final byte[] update = KVParameterRequires.requireNonNull(request.getUpdate(), "compareAndPut.update");
+            this.rawKVStore.compareAndPut(key, expect, update, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleDeleteRequest(final DeleteRequest request,
+                                    final RequestProcessClosure> closure) {
+        final DeleteResponse response = new DeleteResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "delete.key");
+            this.rawKVStore.delete(key, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleDeleteRangeRequest(final DeleteRangeRequest request,
+                                         final RequestProcessClosure> closure) {
+        final DeleteRangeResponse response = new DeleteRangeResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] startKey = KVParameterRequires.requireNonNull(request.getStartKey(), "deleteRange.startKey");
+            final byte[] endKey = KVParameterRequires.requireNonNull(request.getEndKey(), "deleteRange.endKey");
+            this.rawKVStore.deleteRange(startKey, endKey, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleBatchDeleteRequest(final BatchDeleteRequest request,
+                                         final RequestProcessClosure> closure) {
+        final BatchDeleteResponse response = new BatchDeleteResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final List keys = KVParameterRequires.requireNonEmpty(request.getKeys(), "delete.keys");
+            this.rawKVStore.delete(keys, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleMergeRequest(final MergeRequest request,
+                                   final RequestProcessClosure> closure) {
+        final MergeResponse response = new MergeResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "merge.key");
+            final byte[] value = KVParameterRequires.requireNonNull(request.getValue(), "merge.value");
+            this.rawKVStore.merge(key, value, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleGetRequest(final GetRequest request,
+                                 final RequestProcessClosure> closure) {
+        final GetResponse response = new GetResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "get.key");
+            this.rawKVStore.get(key, request.isReadOnlySafe(), new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((byte[]) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleMultiGetRequest(final MultiGetRequest request,
+                                      final RequestProcessClosure> closure) {
+        final MultiGetResponse response = new MultiGetResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final List keys = KVParameterRequires.requireNonEmpty(request.getKeys(), "multiGet.keys");
+            this.rawKVStore.multiGet(keys, request.isReadOnlySafe(), new BaseKVStoreClosure() {
+
+                @SuppressWarnings("unchecked")
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Map) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleContainsKeyRequest(final ContainsKeyRequest request,
+                                         final RequestProcessClosure> closure) {
+        final ContainsKeyResponse response = new ContainsKeyResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "containsKey.key");
+            this.rawKVStore.containsKey(key, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleScanRequest(final ScanRequest request,
+                                  final RequestProcessClosure> closure) {
+        final ScanResponse response = new ScanResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final BaseKVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+
+                @SuppressWarnings("unchecked")
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((List) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            };
+            if (request.isReverse()) {
+                this.rawKVStore.reverseScan(request.getStartKey(), request.getEndKey(), request.getLimit(),
+                    request.isReadOnlySafe(), request.isReturnValue(), kvStoreClosure);
+            } else {
+                this.rawKVStore.scan(request.getStartKey(), request.getEndKey(), request.getLimit(),
+                    request.isReadOnlySafe(), request.isReturnValue(), kvStoreClosure);
+            }
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleGetSequence(final GetSequenceRequest request,
+                                  final RequestProcessClosure> closure) {
+        final GetSequenceResponse response = new GetSequenceResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] seqKey = KVParameterRequires.requireNonNull(request.getSeqKey(), "sequence.seqKey");
+            final int step = KVParameterRequires.requireNonNegative(request.getStep(), "sequence.step");
+            this.rawKVStore.getSequence(seqKey, step, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Sequence) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleResetSequence(final ResetSequenceRequest request,
+                                    final RequestProcessClosure> closure) {
+        final ResetSequenceResponse response = new ResetSequenceResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] seqKey = KVParameterRequires.requireNonNull(request.getSeqKey(), "sequence.seqKey");
+            this.rawKVStore.resetSequence(seqKey, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleKeyLockRequest(final KeyLockRequest request,
+                                     final RequestProcessClosure> closure) {
+        final KeyLockResponse response = new KeyLockResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "lock.key");
+            final byte[] fencingKey = this.regionEngine.getRegion().getStartKey();
+            final DistributedLock.Acquirer acquirer = KVParameterRequires.requireNonNull(request.getAcquirer(),
+                "lock.acquirer");
+            KVParameterRequires.requireNonNull(acquirer.getId(), "lock.id");
+            KVParameterRequires.requirePositive(acquirer.getLeaseMillis(), "lock.leaseMillis");
+            this.rawKVStore.tryLockWith(key, fencingKey, request.isKeepLease(), acquirer, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((DistributedLock.Owner) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleKeyUnlockRequest(final KeyUnlockRequest request,
+                                       final RequestProcessClosure> closure) {
+        final KeyUnlockResponse response = new KeyUnlockResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final byte[] key = KVParameterRequires.requireNonNull(request.getKey(), "unlock.key");
+            final DistributedLock.Acquirer acquirer = KVParameterRequires.requireNonNull(request.getAcquirer(),
+                "lock.acquirer");
+            KVParameterRequires.requireNonNull(acquirer.getId(), "lock.id");
+            this.rawKVStore.releaseLockWith(key, acquirer, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((DistributedLock.Owner) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleNodeExecuteRequest(final NodeExecuteRequest request,
+                                         final RequestProcessClosure> closure) {
+        final NodeExecuteResponse response = new NodeExecuteResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final NodeExecutor executor = KVParameterRequires
+                .requireNonNull(request.getNodeExecutor(), "node.executor");
+            this.rawKVStore.execute(executor, true, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleRangeSplitRequest(final RangeSplitRequest request,
+                                        final RequestProcessClosure> closure) {
+        final RangeSplitResponse response = new RangeSplitResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            // do not need to check the region epoch
+            final Long newRegionId = KVParameterRequires.requireNonNull(request.getNewRegionId(),
+                "rangeSplit.newRegionId");
+            this.regionEngine.getStoreEngine().applySplit(request.getRegionId(), newRegionId, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleCompareAndPutAll(final CASAllRequest request,
+                                       final RequestProcessClosure> closure) {
+        final CASAllResponse response = new CASAllResponse();
+        response.setRegionId(getRegionId());
+        response.setRegionEpoch(getRegionEpoch());
+        try {
+            KVParameterRequires.requireSameEpoch(request, getRegionEpoch());
+            final List casEntries = KVParameterRequires.requireNonEmpty(request.getCasEntries(),
+                "casAll.casEntries");
+            this.rawKVStore.compareAndPutAll(casEntries, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(final Status status) {
+                    if (status.isOk()) {
+                        response.setValue((Boolean) getData());
+                    } else {
+                        setFailure(request, response, status, getError());
+                    }
+                    closure.sendResponse(response);
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+
+    }
+
+    private static void setFailure(final BaseRequest request, final BaseResponse response, final Status status,
+                                   final Errors error) {
+        response.setError(error == null ? Errors.STORAGE_ERROR : error);
+        LOG.error("Failed to handle: {}, status: {}, error: {}.", request, status, error);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DescriberManager.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DescriberManager.java
new file mode 100644
index 0000000..774c0f8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/DescriberManager.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.Describer;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class DescriberManager {
+
+    private static final DescriberManager INSTANCE   = new DescriberManager();
+
+    private final List         describers = new CopyOnWriteArrayList<>();
+
+    public static DescriberManager getInstance() {
+        return INSTANCE;
+    }
+
+    public void addDescriber(final Describer describer) {
+        this.describers.add(describer);
+    }
+
+    public List getAllDescribers() {
+        return Lists.newArrayList(this.describers);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/FollowerStateListener.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/FollowerStateListener.java
new file mode 100644
index 0000000..c6f517d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/FollowerStateListener.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ * Follower state listener.
+ *
+ * @author jiachun.fjc
+ */
+public interface FollowerStateListener extends StateListener {
+
+    /**
+     * Called when current node becomes leader.
+     *
+     * @param term the new term
+     */
+    default void onLeaderStart(final long term) {
+        // NO-OP
+    }
+
+    /**
+     * Called when current node loses leadership.
+     *
+     * @param term the old term
+     */
+    default void onLeaderStop(final long term) {
+        // NO-OP
+    }
+
+    /**
+     * This method is called when a follower or candidate starts following a leader and its leaderId
+     * (should be NULL before the method is called) is set to the leader's id, situations including:
+     * 1. A candidate receives appendEntries request from a leader
+     * 2. A follower(without leader) receives appendEntries from a leader
+     *
+     * The parameters gives the information(leaderId and term) about the very
+     * leader whom the follower starts to follow.
+     * User can reset the node's information as it starts to follow some leader.
+     *
+     * @param newLeaderId the new leader id whom the follower starts to follow
+     * @param newTerm     the new term
+     */
+    void onStartFollowing(final PeerId newLeaderId, final long newTerm);
+
+    /**
+     * This method is called when a follower stops following a leader and its leaderId becomes null,
+     * situations including:
+     * 1. Handle election timeout and start preVote
+     * 2. Receive requests with higher term such as VoteRequest from a candidate
+     *      or appendEntries request from a new leader
+     * 3. Receive timeoutNow request from current leader and start request vote.
+     *
+     * The parameters gives the information(leaderId and term) about the very leader
+     * whom the follower followed before.
+     * User can reset the node's information as it stops following some leader.
+     *
+     * @param oldLeaderId the old leader id whom the follower followed before
+     * @param oldTerm     the old term
+     */
+    void onStopFollowing(final PeerId oldLeaderId, final long oldTerm);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/JRaftHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/JRaftHelper.java
new file mode 100644
index 0000000..c8a86b8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/JRaftHelper.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class JRaftHelper {
+
+    public static String getJRaftGroupId(final String clusterName, final long regionId) {
+        Requires.requireNonNull(clusterName, "clusterName");
+        return clusterName + "-" + regionId;
+    }
+
+    public static PeerId toJRaftPeerId(final Peer peer) {
+        Requires.requireNonNull(peer, "peer");
+        final Endpoint endpoint = peer.getEndpoint();
+        Requires.requireNonNull(endpoint, "peer.endpoint");
+        return new PeerId(endpoint, 0);
+    }
+
+    public static List toJRaftPeerIdList(final List peerList) {
+        if (peerList == null) {
+            return null;
+        }
+        final List peerIdList = Lists.newArrayListWithCapacity(peerList.size());
+        for (final Peer peer : peerList) {
+            peerIdList.add(toJRaftPeerId(peer));
+        }
+        return peerIdList;
+    }
+
+    public static Peer toPeer(final PeerId peerId) {
+        Requires.requireNonNull(peerId, "peerId");
+        final Endpoint endpoint = peerId.getEndpoint();
+        Requires.requireNonNull(endpoint, "peerId.endpoint");
+        final Peer peer = new Peer();
+        peer.setId(-1);
+        peer.setStoreId(-1);
+        peer.setEndpoint(endpoint.copy());
+        return peer;
+    }
+
+    public static List toPeerList(final List peerIdList) {
+        if (peerIdList == null) {
+            return null;
+        }
+        final List peerList = Lists.newArrayListWithCapacity(peerIdList.size());
+        for (final PeerId peerId : peerIdList) {
+            peerList.add(toPeer(peerId));
+        }
+        return peerList;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/KVCommandProcessor.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/KVCommandProcessor.java
new file mode 100644
index 0000000..6565122
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/KVCommandProcessor.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.concurrent.Executor;
+
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.NoRegionFoundResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.RheaRuntimeException;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * Rhea KV store RPC request processing service.
+ *
+ * @author jiachun.fjc
+ */
+public class KVCommandProcessor implements RpcProcessor {
+
+    private final Class    reqClazz;
+    private final StoreEngine storeEngine;
+
+    public KVCommandProcessor(Class reqClazz, StoreEngine storeEngine) {
+        this.reqClazz = Requires.requireNonNull(reqClazz, "reqClazz");
+        this.storeEngine = Requires.requireNonNull(storeEngine, "storeEngine");
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final T request) {
+        Requires.requireNonNull(request, "request");
+        final RequestProcessClosure> closure = new RequestProcessClosure<>(request, rpcCtx);
+        final RegionKVService regionKVService = this.storeEngine.getRegionKVService(request.getRegionId());
+        if (regionKVService == null) {
+            final NoRegionFoundResponse noRegion = new NoRegionFoundResponse();
+            noRegion.setRegionId(request.getRegionId());
+            noRegion.setError(Errors.NO_REGION_FOUND);
+            noRegion.setValue(false);
+            closure.sendResponse(noRegion);
+            return;
+        }
+        switch (request.magic()) {
+            case BaseRequest.PUT:
+                regionKVService.handlePutRequest((PutRequest) request, closure);
+                break;
+            case BaseRequest.BATCH_PUT:
+                regionKVService.handleBatchPutRequest((BatchPutRequest) request, closure);
+                break;
+            case BaseRequest.PUT_IF_ABSENT:
+                regionKVService.handlePutIfAbsentRequest((PutIfAbsentRequest) request, closure);
+                break;
+            case BaseRequest.GET_PUT:
+                regionKVService.handleGetAndPutRequest((GetAndPutRequest) request, closure);
+                break;
+            case BaseRequest.COMPARE_PUT:
+                regionKVService.handleCompareAndPutRequest((CompareAndPutRequest) request, closure);
+                break;
+            case BaseRequest.DELETE:
+                regionKVService.handleDeleteRequest((DeleteRequest) request, closure);
+                break;
+            case BaseRequest.DELETE_RANGE:
+                regionKVService.handleDeleteRangeRequest((DeleteRangeRequest) request, closure);
+                break;
+            case BaseRequest.BATCH_DELETE:
+                regionKVService.handleBatchDeleteRequest((BatchDeleteRequest) request, closure);
+                break;
+            case BaseRequest.MERGE:
+                regionKVService.handleMergeRequest((MergeRequest) request, closure);
+                break;
+            case BaseRequest.GET:
+                regionKVService.handleGetRequest((GetRequest) request, closure);
+                break;
+            case BaseRequest.MULTI_GET:
+                regionKVService.handleMultiGetRequest((MultiGetRequest) request, closure);
+                break;
+            case BaseRequest.CONTAINS_KEY:
+                regionKVService.handleContainsKeyRequest((ContainsKeyRequest) request, closure);
+                break;
+            case BaseRequest.SCAN:
+                regionKVService.handleScanRequest((ScanRequest) request, closure);
+                break;
+            case BaseRequest.GET_SEQUENCE:
+                regionKVService.handleGetSequence((GetSequenceRequest) request, closure);
+                break;
+            case BaseRequest.RESET_SEQUENCE:
+                regionKVService.handleResetSequence((ResetSequenceRequest) request, closure);
+                break;
+            case BaseRequest.KEY_LOCK:
+                regionKVService.handleKeyLockRequest((KeyLockRequest) request, closure);
+                break;
+            case BaseRequest.KEY_UNLOCK:
+                regionKVService.handleKeyUnlockRequest((KeyUnlockRequest) request, closure);
+                break;
+            case BaseRequest.NODE_EXECUTE:
+                regionKVService.handleNodeExecuteRequest((NodeExecuteRequest) request, closure);
+                break;
+            case BaseRequest.RANGE_SPLIT:
+                regionKVService.handleRangeSplitRequest((RangeSplitRequest) request, closure);
+                break;
+            case BaseRequest.COMPARE_PUT_ALL:
+                regionKVService.handleCompareAndPutAll((CASAllRequest) request, closure);
+                break;
+            default:
+                throw new RheaRuntimeException("Unsupported request type: " + request.getClass().getName());
+        }
+    }
+
+    @Override
+    public String interest() {
+        return this.reqClazz.getName();
+    }
+
+    @Override
+    public Executor executor() {
+        return this.storeEngine.getKvRpcExecutor();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/LeaderStateListener.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/LeaderStateListener.java
new file mode 100644
index 0000000..91247a4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/LeaderStateListener.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ * Leader state listener.
+ *
+ * @author dennis
+ */
+public interface LeaderStateListener extends StateListener {
+
+    /**
+     * Called when current node becomes leader.
+     *
+     * @param newTerm the new term
+     */
+    void onLeaderStart(final long newTerm);
+
+    /**
+     * Called when current node loses leadership.
+     *
+     * @param oldTerm the old term
+     */
+    void onLeaderStop(final long oldTerm);
+
+    /**
+     * This method is called when a follower or candidate starts following a leader and its leaderId
+     * (should be NULL before the method is called) is set to the leader's id, situations including:
+     * 1. A candidate receives appendEntries request from a leader
+     * 2. A follower(without leader) receives appendEntries from a leader
+     *
+     * The parameters gives the information(leaderId and term) about the very
+     * leader whom the follower starts to follow.
+     * User can reset the node's information as it starts to follow some leader.
+     *
+     * @param newLeaderId the new leader id whom the follower starts to follow
+     * @param newTerm     the new term
+     */
+    default void onStartFollowing(final PeerId newLeaderId, final long newTerm) {
+        // NO-OP
+    }
+
+    /**
+     * This method is called when a follower stops following a leader and its leaderId becomes null,
+     * situations including:
+     * 1. Handle election timeout and start preVote
+     * 2. Receive requests with higher term such as VoteRequest from a candidate
+     *      or appendEntries request from a new leader
+     * 3. Receive timeoutNow request from current leader and start request vote.
+     *
+     * The parameters gives the information(leaderId and term) about the very leader
+     * whom the follower followed before.
+     * User can reset the node's information as it stops following some leader.
+     *
+     * @param oldLeaderId the old leader id whom the follower followed before
+     * @param oldTerm     the old term
+     */
+    default void onStopFollowing(final PeerId oldLeaderId, final long oldTerm) {
+        // NO-OP
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionEngine.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionEngine.java
new file mode 100644
index 0000000..12aa4cb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionEngine.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreStateMachine;
+import com.alipay.sofa.jraft.rhea.storage.MetricsRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RaftRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RawKVStore;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.Describer;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+
+/**
+ * Minimum execution/copy unit of RheaKVStore.
+ *
+ * @author jiachun.fjc
+ */
+public class RegionEngine implements Lifecycle, Describer {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RegionEngine.class);
+
+    private final Region        region;
+    private final StoreEngine   storeEngine;
+
+    private RaftRawKVStore      raftRawKVStore;
+    private MetricsRawKVStore   metricsRawKVStore;
+    private RaftGroupService    raftGroupService;
+    private Node                node;
+    private KVStoreStateMachine fsm;
+    private RegionEngineOptions regionOpts;
+
+    private ScheduledReporter   regionMetricsReporter;
+
+    private boolean             started;
+
+    public RegionEngine(Region region, StoreEngine storeEngine) {
+        this.region = region;
+        this.storeEngine = storeEngine;
+    }
+
+    @Override
+    public synchronized boolean init(final RegionEngineOptions opts) {
+        if (this.started) {
+            LOG.info("[RegionEngine: {}] already started.", this.region);
+            return true;
+        }
+        this.regionOpts = Requires.requireNonNull(opts, "opts");
+        this.fsm = new KVStoreStateMachine(this.region, this.storeEngine);
+
+        // node options
+        NodeOptions nodeOpts = opts.getNodeOptions();
+        if (nodeOpts == null) {
+            nodeOpts = new NodeOptions();
+        }
+        final long metricsReportPeriod = opts.getMetricsReportPeriod();
+        if (metricsReportPeriod > 0) {
+            // metricsReportPeriod > 0 means enable metrics
+            nodeOpts.setEnableMetrics(true);
+        }
+        final Configuration initialConf = new Configuration();
+        if (!initialConf.parse(opts.getInitialServerList())) {
+            LOG.error("Fail to parse initial configuration {}.", opts.getInitialServerList());
+            return false;
+        }
+        nodeOpts.setInitialConf(initialConf);
+        nodeOpts.setFsm(this.fsm);
+        final String raftDataPath = opts.getRaftDataPath();
+        try {
+            FileUtils.forceMkdir(new File(raftDataPath));
+        } catch (final Throwable t) {
+            LOG.error("Fail to make dir for raftDataPath {}.", raftDataPath);
+            return false;
+        }
+        if (Strings.isBlank(nodeOpts.getLogUri())) {
+            final Path logUri = Paths.get(raftDataPath, "log");
+            nodeOpts.setLogUri(logUri.toString());
+        }
+        if (Strings.isBlank(nodeOpts.getRaftMetaUri())) {
+            final Path meteUri = Paths.get(raftDataPath, "meta");
+            nodeOpts.setRaftMetaUri(meteUri.toString());
+        }
+        if (Strings.isBlank(nodeOpts.getSnapshotUri())) {
+            final Path snapshotUri = Paths.get(raftDataPath, "snapshot");
+            nodeOpts.setSnapshotUri(snapshotUri.toString());
+        }
+        LOG.info("[RegionEngine: {}], log uri: {}, raft meta uri: {}, snapshot uri: {}.", this.region,
+            nodeOpts.getLogUri(), nodeOpts.getRaftMetaUri(), nodeOpts.getSnapshotUri());
+        final Endpoint serverAddress = opts.getServerAddress();
+        final PeerId serverId = new PeerId(serverAddress, 0);
+        final RpcServer rpcServer = this.storeEngine.getRpcServer();
+        this.raftGroupService = new RaftGroupService(opts.getRaftGroupId(), serverId, nodeOpts, rpcServer, true);
+        this.node = this.raftGroupService.start(false);
+        RouteTable.getInstance().updateConfiguration(this.raftGroupService.getGroupId(), nodeOpts.getInitialConf());
+        if (this.node != null) {
+            final RawKVStore rawKVStore = this.storeEngine.getRawKVStore();
+            final Executor readIndexExecutor = this.storeEngine.getReadIndexExecutor();
+            this.raftRawKVStore = new RaftRawKVStore(this.node, rawKVStore, readIndexExecutor);
+            this.metricsRawKVStore = new MetricsRawKVStore(this.region.getId(), this.raftRawKVStore);
+            // metrics config
+            if (this.regionMetricsReporter == null && metricsReportPeriod > 0) {
+                final MetricRegistry metricRegistry = this.node.getNodeMetrics().getMetricRegistry();
+                if (metricRegistry != null) {
+                    final ScheduledExecutorService scheduler = this.storeEngine.getMetricsScheduler();
+                    // start raft node metrics reporter
+                    this.regionMetricsReporter = Slf4jReporter.forRegistry(metricRegistry) //
+                        .prefixedWith("region_" + this.region.getId()) //
+                        .withLoggingLevel(Slf4jReporter.LoggingLevel.INFO) //
+                        .outputTo(LOG) //
+                        .scheduleOn(scheduler) //
+                        .shutdownExecutorOnStop(scheduler != null) //
+                        .build();
+                    this.regionMetricsReporter.start(metricsReportPeriod, TimeUnit.SECONDS);
+                }
+            }
+            this.started = true;
+            LOG.info("[RegionEngine] start successfully: {}.", this);
+        }
+        return this.started;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        if (this.raftGroupService != null) {
+            this.raftGroupService.shutdown();
+            try {
+                this.raftGroupService.join();
+            } catch (final InterruptedException e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        if (this.regionMetricsReporter != null) {
+            this.regionMetricsReporter.stop();
+        }
+        this.started = false;
+        LOG.info("[RegionEngine] shutdown successfully: {}.", this);
+    }
+
+    public boolean transferLeadershipTo(final Endpoint endpoint) {
+        final PeerId peerId = new PeerId(endpoint, 0);
+        final Status status = this.node.transferLeadershipTo(peerId);
+        final boolean isOk = status.isOk();
+        if (isOk) {
+            LOG.info("Transfer-leadership succeeded: [{} --> {}].", this.storeEngine.getSelfEndpoint(), endpoint);
+        } else {
+            LOG.error("Transfer-leadership failed: {}, [{} --> {}].", status, this.storeEngine.getSelfEndpoint(),
+                endpoint);
+        }
+        return isOk;
+    }
+
+    public Region getRegion() {
+        return region;
+    }
+
+    public StoreEngine getStoreEngine() {
+        return storeEngine;
+    }
+
+    public boolean isLeader() {
+        return this.node.isLeader(false);
+    }
+
+    public PeerId getLeaderId() {
+        return this.node.getLeaderId();
+    }
+
+    public RaftRawKVStore getRaftRawKVStore() {
+        return raftRawKVStore;
+    }
+
+    public MetricsRawKVStore getMetricsRawKVStore() {
+        return metricsRawKVStore;
+    }
+
+    public Node getNode() {
+        return node;
+    }
+
+    public KVStoreStateMachine getFsm() {
+        return fsm;
+    }
+
+    public RegionEngineOptions copyRegionOpts() {
+        return Requires.requireNonNull(this.regionOpts, "opts").copy();
+    }
+
+    @Override
+    public String toString() {
+        return "RegionEngine{" + "region=" + region + ", isLeader=" + isLeader() + ", regionOpts=" + regionOpts + '}';
+    }
+
+    @Override
+    public void describe(final Printer out) {
+        out.print("  RegionEngine: ") //
+            .print("regionId=") //
+            .print(this.region.getId()) //
+            .print(", isLeader=") //
+            .print(isLeader()) //
+            .print(", leaderId=") //
+            .println(getLeaderId());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionKVService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionKVService.java
new file mode 100644
index 0000000..9341a64
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RegionKVService.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanRequest;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+
+/**
+ * Request processing service on the KV server side.
+ * 

+ * A {@link StoreEngine} contains many {@link RegionKVService}s,
+ * each {@link RegionKVService} corresponds to a region, and it
+ * only processes request keys within its own region.
+ *
+ * @author jiachun.fjc
+ */
+public interface RegionKVService {
+
+    long getRegionId();
+
+    RegionEpoch getRegionEpoch();
+
+    /**
+     * {@link BaseRequest#PUT}
+     */
+    void handlePutRequest(final PutRequest request, final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#BATCH_PUT}
+     */
+    void handleBatchPutRequest(final BatchPutRequest request,
+                               final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#PUT_IF_ABSENT}
+     */
+    void handlePutIfAbsentRequest(final PutIfAbsentRequest request,
+                                  final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#GET_PUT}
+     */
+    void handleGetAndPutRequest(final GetAndPutRequest request,
+                                final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#COMPARE_PUT}
+     */
+    void handleCompareAndPutRequest(final CompareAndPutRequest request,
+                                    final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#DELETE}
+     */
+    void handleDeleteRequest(final DeleteRequest request,
+                             final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#DELETE_RANGE}
+     */
+    void handleDeleteRangeRequest(final DeleteRangeRequest request,
+                                  final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#BATCH_DELETE}
+     */
+    void handleBatchDeleteRequest(final BatchDeleteRequest request,
+                                  final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#MERGE}
+     */
+    void handleMergeRequest(final MergeRequest request,
+                            final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#GET}
+     */
+    void handleGetRequest(final GetRequest request, final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#MULTI_GET}
+     */
+    void handleMultiGetRequest(final MultiGetRequest request,
+                               final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#CONTAINS_KEY}
+     */
+    void handleContainsKeyRequest(final ContainsKeyRequest request,
+                                  final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#SCAN}
+     */
+    void handleScanRequest(final ScanRequest request, final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#GET_SEQUENCE}
+     */
+    void handleGetSequence(final GetSequenceRequest request,
+                           final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#RESET_SEQUENCE}
+     */
+    void handleResetSequence(final ResetSequenceRequest request,
+                             final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#KEY_LOCK}
+     */
+    void handleKeyLockRequest(final KeyLockRequest request,
+                              final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#KEY_UNLOCK}
+     */
+    void handleKeyUnlockRequest(final KeyUnlockRequest request,
+                                final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#NODE_EXECUTE}
+     */
+    void handleNodeExecuteRequest(final NodeExecuteRequest request,
+                                  final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#RANGE_SPLIT}
+     */
+    void handleRangeSplitRequest(final RangeSplitRequest request,
+                                 final RequestProcessClosure> closure);
+
+    /**
+     * {@link BaseRequest#COMPARE_PUT_ALL}
+     */
+    void handleCompareAndPutAll(final CASAllRequest request,
+                                final RequestProcessClosure> closure);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RequestProcessClosure.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RequestProcessClosure.java
new file mode 100644
index 0000000..0cb9afd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RequestProcessClosure.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+
+/**
+ * RPC request processor closure wraps request/response and network biz context.
+ *
+ * @author dennis
+ * @author jiachun.fjc
+ */
+public class RequestProcessClosure implements Closure {
+
+    private static final Logger                                           LOG           = LoggerFactory
+                                                                                            .getLogger(RequestProcessClosure.class);
+
+    private static final AtomicIntegerFieldUpdater STATE_UPDATER = AtomicIntegerFieldUpdater
+                                                                                            .newUpdater(
+                                                                                                RequestProcessClosure.class,
+                                                                                                "state");
+
+    private static final int                                              PENDING       = 0;
+    private static final int                                              RESPOND       = 1;
+
+    private final REQ                                                     request;
+    private final RpcContext                                              rpcCtx;
+
+    private RSP                                                           response;
+
+    private volatile int                                                  state         = PENDING;
+
+    public RequestProcessClosure(REQ request, RpcContext rpcCtx) {
+        super();
+        this.request = request;
+        this.rpcCtx = rpcCtx;
+    }
+
+    public RpcContext getRpcCtx() {
+        return rpcCtx;
+    }
+
+    public REQ getRequest() {
+        return request;
+    }
+
+    public RSP getResponse() {
+        return response;
+    }
+
+    public void sendResponse(final RSP response) {
+        this.response = response;
+        run(null);
+    }
+
+    /**
+     * Run the closure and send response.
+     */
+    @Override
+    public void run(final Status status) {
+        if (!STATE_UPDATER.compareAndSet(this, PENDING, RESPOND)) {
+            LOG.warn("A response: {} with status: {} sent repeatedly!", this.response, status);
+            return;
+        }
+        this.rpcCtx.sendResponse(this.response);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVDescribeSignalHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVDescribeSignalHandler.java
new file mode 100644
index 0000000..d20b0da
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVDescribeSignalHandler.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.util.Describer;
+import com.alipay.sofa.jraft.util.FileOutputSignalHandler;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVDescribeSignalHandler extends FileOutputSignalHandler {
+
+    private static Logger       LOG       = LoggerFactory.getLogger(RheaKVDescribeSignalHandler.class);
+
+    private static final String DIR       = SystemPropertyUtil.get("rheakv.signal.describe.dir", "");
+    private static final String BASE_NAME = "rheakv_describe.log";
+
+    @Override
+    public void handle(final String signalName) {
+        final List describers = DescriberManager.getInstance().getAllDescribers();
+        if (describers.isEmpty()) {
+            return;
+        }
+
+        try {
+            final File file = getOutputFile(DIR, BASE_NAME);
+
+            LOG.info("Describing rheakv with signal: {} to file: {}.", signalName, file);
+
+            try (final PrintWriter out = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file, true),
+                StandardCharsets.UTF_8))) {
+                final Describer.Printer printer = new Describer.DefaultPrinter(out);
+                for (final Describer describer : describers) {
+                    describer.describe(printer);
+                }
+            }
+        } catch (final IOException e) {
+            LOG.error("Fail to describe rheakv: {}.", describers, e);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVMetricsSignalHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVMetricsSignalHandler.java
new file mode 100644
index 0000000..d48d48e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVMetricsSignalHandler.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.util.FileOutputSignalHandler;
+import com.alipay.sofa.jraft.util.MetricReporter;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVMetricsSignalHandler extends FileOutputSignalHandler {
+
+    private static Logger       LOG       = LoggerFactory.getLogger(RheaKVMetricsSignalHandler.class);
+
+    private static final String DIR       = SystemPropertyUtil.get("rheakv.signal.metrics.dir", "");
+    private static final String BASE_NAME = "rheakv_metrics.log";
+
+    @Override
+    public void handle(final String signalName) {
+        try {
+            final File file = getOutputFile(DIR, BASE_NAME);
+
+            LOG.info("Printing rheakv metrics with signal: {} to file: {}.", signalName, file);
+
+            try (final PrintStream out = new PrintStream(new FileOutputStream(file, true))) {
+                final MetricReporter reporter = MetricReporter.forRegistry(KVMetrics.metricRegistry()) //
+                    .outputTo(out) //
+                    .prefixedWith("-- rheakv") //
+                    .build();
+                reporter.report();
+            }
+        } catch (final IOException e) {
+            LOG.error("Fail to print rheakv metrics.", e);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVServiceFactory.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVServiceFactory.java
new file mode 100644
index 0000000..02cc095
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/RheaKVServiceFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVCliService;
+import com.alipay.sofa.jraft.rhea.client.RheaKVCliService;
+
+/**
+ * Service factory to create rheaKV services, such as RheaKVCliService etc.
+ *
+ * @author jiachun.fjc
+ */
+public final class RheaKVServiceFactory {
+
+    /**
+     * Create and initialize a RheaKVCliService instance.
+     */
+    public static RheaKVCliService createAndInitRheaKVCliService(final CliOptions opts) {
+        final RheaKVCliService cliService = new DefaultRheaKVCliService();
+        if (!cliService.init(opts)) {
+            throw new IllegalStateException("Fail to init RheaKVCliService");
+        }
+        return cliService;
+    }
+
+    private RheaKVServiceFactory() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListener.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListener.java
new file mode 100644
index 0000000..acd1921
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListener.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ * The raft state listener.
+ *
+ * @author jiachun.fjc
+ */
+public interface StateListener {
+
+    /**
+     * Called when current node becomes leader.
+     *
+     * @param newTerm the new term
+     */
+    void onLeaderStart(final long newTerm);
+
+    /**
+     * Called when current node loses leadership.
+     *
+     * @param oldTerm the old term
+     */
+    void onLeaderStop(final long oldTerm);
+
+    /**
+     * This method is called when a follower or candidate starts following a leader and its leaderId
+     * (should be NULL before the method is called) is set to the leader's id, situations including:
+     * 1. A candidate receives appendEntries request from a leader
+     * 2. A follower(without leader) receives appendEntries from a leader
+     *
+     * The parameters gives the information(leaderId and term) about the very
+     * leader whom the follower starts to follow.
+     * User can reset the node's information as it starts to follow some leader.
+     *
+     * @param newLeaderId the new leader id whom the follower starts to follow
+     * @param newTerm     the new term
+     */
+    void onStartFollowing(final PeerId newLeaderId, final long newTerm);
+
+    /**
+     * This method is called when a follower stops following a leader and its leaderId becomes null,
+     * situations including:
+     * 1. Handle election timeout and start preVote
+     * 2. Receive requests with higher term such as VoteRequest from a candidate
+     *      or appendEntries request from a new leader
+     * 3. Receive timeoutNow request from current leader and start request vote.
+     *
+     * The parameters gives the information(leaderId and term) about the very leader
+     * whom the follower followed before.
+     * User can reset the node's information as it stops following some leader.
+     *
+     * @param oldLeaderId the old leader id whom the follower followed before
+     * @param oldTerm     the old term
+     */
+    void onStopFollowing(final PeerId oldLeaderId, final long oldTerm);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListenerContainer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListenerContainer.java
new file mode 100644
index 0000000..acb013d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StateListenerContainer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ * The container of raft state listener, each key(id) corresponds to a listener group.
+ *
+ * @author jiachun.fjc
+ */
+public class StateListenerContainer {
+
+    private final ConcurrentMap> stateListeners = Maps.newConcurrentMap();
+
+    public boolean addStateListener(final K id, final StateListener listener) {
+        List group = this.stateListeners.get(id);
+        if (group == null) {
+            final List newGroup = new CopyOnWriteArrayList<>();
+            group = this.stateListeners.putIfAbsent(id, newGroup);
+            if (group == null) {
+                group = newGroup;
+            }
+        }
+        return group.add(listener);
+    }
+
+    public List getStateListenerGroup(final K id) {
+        final List group = this.stateListeners.get(id);
+        return group == null ? Collections.emptyList() : group;
+    }
+
+    public boolean removeStateListener(final K id, final StateListener listener) {
+        final List group = this.stateListeners.get(id);
+        if (group == null) {
+            return false;
+        }
+        return group.remove(listener);
+    }
+
+    public void clear() {
+        this.stateListeners.clear();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngine.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngine.java
new file mode 100644
index 0000000..c5a84fd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngine.java
@@ -0,0 +1,732 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.client.pd.HeartbeatSender;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.client.pd.RemotePlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.RheaRuntimeException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.options.HeartbeatOptions;
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.rpc.ExtSerializerSupports;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.storage.BatchRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVClosureAdapter;
+import com.alipay.sofa.jraft.rhea.storage.KVOperation;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.Constants;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.NetUtil;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Describer;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolMetricRegistry;
+import com.alipay.sofa.jraft.util.Utils;
+import com.codahale.metrics.ScheduledReporter;
+import com.codahale.metrics.Slf4jReporter;
+
+/**
+ * Storage engine, there is only one instance in a node,
+ * containing one or more {@link RegionEngine}.
+ *
+ * @author jiachun.fjc
+ */
+public class StoreEngine implements Lifecycle, Describer {
+
+    private static final Logger                        LOG                  = LoggerFactory
+                                                                                .getLogger(StoreEngine.class);
+
+    static {
+        ExtSerializerSupports.init();
+    }
+
+    private final ConcurrentMap regionKVServiceTable = Maps.newConcurrentMapLong();
+    private final ConcurrentMap    regionEngineTable    = Maps.newConcurrentMapLong();
+    private final StateListenerContainer         stateListenerContainer;
+    private final PlacementDriverClient                pdClient;
+    private final long                                 clusterId;
+
+    private Long                                       storeId;
+    private final AtomicBoolean                        splitting            = new AtomicBoolean(false);
+    // When the store is started (unix timestamp in milliseconds)
+    private long                                       startTime            = System.currentTimeMillis();
+    private File                                       dbPath;
+    private RpcServer                                  rpcServer;
+    private BatchRawKVStore                         rawKVStore;
+    private HeartbeatSender                            heartbeatSender;
+    private StoreEngineOptions                         storeOpts;
+
+    // Shared executor services
+    private ExecutorService                            readIndexExecutor;
+    private ExecutorService                            raftStateTrigger;
+    private ExecutorService                            snapshotExecutor;
+    private ExecutorService                            cliRpcExecutor;
+    private ExecutorService                            raftRpcExecutor;
+    private ExecutorService                            kvRpcExecutor;
+
+    private ScheduledExecutorService                   metricsScheduler;
+    private ScheduledReporter                          kvMetricsReporter;
+    private ScheduledReporter                          threadPoolMetricsReporter;
+
+    private boolean                                    started;
+
+    public StoreEngine(PlacementDriverClient pdClient, StateListenerContainer stateListenerContainer) {
+        this.pdClient = Requires.requireNonNull(pdClient, "pdClient");
+        this.clusterId = pdClient.getClusterId();
+        this.stateListenerContainer = Requires.requireNonNull(stateListenerContainer, "stateListenerContainer");
+    }
+
+    @Override
+    public synchronized boolean init(final StoreEngineOptions opts) {
+        if (this.started) {
+            LOG.info("[StoreEngine] already started.");
+            return true;
+        }
+
+        DescriberManager.getInstance().addDescriber(this);
+
+        this.storeOpts = Requires.requireNonNull(opts, "opts");
+        Endpoint serverAddress = Requires.requireNonNull(opts.getServerAddress(), "opts.serverAddress");
+        final int port = serverAddress.getPort();
+        final String ip = serverAddress.getIp();
+        if (ip == null || Utils.IP_ANY.equals(ip)) {
+            serverAddress = new Endpoint(NetUtil.getLocalCanonicalHostName(), port);
+            opts.setServerAddress(serverAddress);
+        }
+        final long metricsReportPeriod = opts.getMetricsReportPeriod();
+        // init region options
+        List rOptsList = opts.getRegionEngineOptionsList();
+        if (rOptsList == null || rOptsList.isEmpty()) {
+            // -1 region
+            final RegionEngineOptions rOpts = new RegionEngineOptions();
+            rOpts.setRegionId(Constants.DEFAULT_REGION_ID);
+            rOptsList = Lists.newArrayList();
+            rOptsList.add(rOpts);
+            opts.setRegionEngineOptionsList(rOptsList);
+        }
+        final String clusterName = this.pdClient.getClusterName();
+        for (final RegionEngineOptions rOpts : rOptsList) {
+            rOpts.setRaftGroupId(JRaftHelper.getJRaftGroupId(clusterName, rOpts.getRegionId()));
+            rOpts.setServerAddress(serverAddress);
+            if (Strings.isBlank(rOpts.getInitialServerList())) {
+                // if blank, extends parent's value
+                rOpts.setInitialServerList(opts.getInitialServerList());
+            }
+            if (rOpts.getNodeOptions() == null) {
+                // copy common node options
+                rOpts.setNodeOptions(opts.getCommonNodeOptions() == null ? new NodeOptions() : opts
+                    .getCommonNodeOptions().copy());
+            }
+            if (rOpts.getMetricsReportPeriod() <= 0 && metricsReportPeriod > 0) {
+                // extends store opts
+                rOpts.setMetricsReportPeriod(metricsReportPeriod);
+            }
+        }
+        // init store
+        final Store store = this.pdClient.getStoreMetadata(opts);
+        if (store == null || store.getRegions() == null || store.getRegions().isEmpty()) {
+            LOG.error("Empty store metadata: {}.", store);
+            return false;
+        }
+        this.storeId = store.getId();
+        // init executors
+        if (this.readIndexExecutor == null) {
+            this.readIndexExecutor = StoreEngineHelper.createReadIndexExecutor(opts.getReadIndexCoreThreads());
+        }
+        if (this.raftStateTrigger == null) {
+            this.raftStateTrigger = StoreEngineHelper.createRaftStateTrigger(opts.getLeaderStateTriggerCoreThreads());
+        }
+        if (this.snapshotExecutor == null) {
+            this.snapshotExecutor = StoreEngineHelper.createSnapshotExecutor(opts.getSnapshotCoreThreads(),
+                opts.getSnapshotMaxThreads());
+        }
+        // init rpc executors
+        final boolean useSharedRpcExecutor = opts.isUseSharedRpcExecutor();
+        if (!useSharedRpcExecutor) {
+            if (this.cliRpcExecutor == null) {
+                this.cliRpcExecutor = StoreEngineHelper.createCliRpcExecutor(opts.getCliRpcCoreThreads());
+            }
+            if (this.raftRpcExecutor == null) {
+                this.raftRpcExecutor = StoreEngineHelper.createRaftRpcExecutor(opts.getRaftRpcCoreThreads());
+            }
+            if (this.kvRpcExecutor == null) {
+                this.kvRpcExecutor = StoreEngineHelper.createKvRpcExecutor(opts.getKvRpcCoreThreads());
+            }
+        }
+        // init metrics
+        startMetricReporters(metricsReportPeriod);
+        // init rpc server
+        this.rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverAddress, this.raftRpcExecutor,
+            this.cliRpcExecutor);
+        StoreEngineHelper.addKvStoreRequestProcessor(this.rpcServer, this);
+        if (!this.rpcServer.init(null)) {
+            LOG.error("Fail to init [RpcServer].");
+            return false;
+        }
+        // init db store
+        if (!initRawKVStore(opts)) {
+            return false;
+        }
+        if (this.rawKVStore instanceof Describer) {
+            DescriberManager.getInstance().addDescriber((Describer) this.rawKVStore);
+        }
+        // init all region engine
+        if (!initAllRegionEngine(opts, store)) {
+            LOG.error("Fail to init all [RegionEngine].");
+            return false;
+        }
+        // heartbeat sender
+        if (this.pdClient instanceof RemotePlacementDriverClient) {
+            HeartbeatOptions heartbeatOpts = opts.getHeartbeatOptions();
+            if (heartbeatOpts == null) {
+                heartbeatOpts = new HeartbeatOptions();
+            }
+            this.heartbeatSender = new HeartbeatSender(this);
+            if (!this.heartbeatSender.init(heartbeatOpts)) {
+                LOG.error("Fail to init [HeartbeatSender].");
+                return false;
+            }
+        }
+        this.startTime = System.currentTimeMillis();
+        LOG.info("[StoreEngine] start successfully: {}.", this);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        if (this.rpcServer != null) {
+            this.rpcServer.shutdown();
+        }
+        if (!this.regionEngineTable.isEmpty()) {
+            for (final RegionEngine engine : this.regionEngineTable.values()) {
+                engine.shutdown();
+            }
+            this.regionEngineTable.clear();
+        }
+        if (this.rawKVStore != null) {
+            this.rawKVStore.shutdown();
+        }
+        if (this.heartbeatSender != null) {
+            this.heartbeatSender.shutdown();
+        }
+        this.regionKVServiceTable.clear();
+        if (this.kvMetricsReporter != null) {
+            this.kvMetricsReporter.stop();
+        }
+        if (this.threadPoolMetricsReporter != null) {
+            this.threadPoolMetricsReporter.stop();
+        }
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.readIndexExecutor);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.raftStateTrigger);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.snapshotExecutor);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.cliRpcExecutor);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.raftRpcExecutor);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.kvRpcExecutor);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.metricsScheduler);
+        this.started = false;
+        LOG.info("[StoreEngine] shutdown successfully.");
+    }
+
+    public PlacementDriverClient getPlacementDriverClient() {
+        return pdClient;
+    }
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public Long getStoreId() {
+        return storeId;
+    }
+
+    public StoreEngineOptions getStoreOpts() {
+        return storeOpts;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public RpcServer getRpcServer() {
+        return rpcServer;
+    }
+
+    public BatchRawKVStore getRawKVStore() {
+        return rawKVStore;
+    }
+
+    public RegionKVService getRegionKVService(final long regionId) {
+        return this.regionKVServiceTable.get(regionId);
+    }
+
+    public long getTotalSpace() {
+        if (this.dbPath == null || !this.dbPath.exists()) {
+            return 0;
+        }
+        return this.dbPath.getTotalSpace();
+    }
+
+    public long getUsableSpace() {
+        if (this.dbPath == null || !this.dbPath.exists()) {
+            return 0;
+        }
+        return this.dbPath.getUsableSpace();
+    }
+
+    public long getStoreUsedSpace() {
+        if (this.dbPath == null || !this.dbPath.exists()) {
+            return 0;
+        }
+        return FileUtils.sizeOf(this.dbPath);
+    }
+
+    public Endpoint getSelfEndpoint() {
+        return this.storeOpts == null ? null : this.storeOpts.getServerAddress();
+    }
+
+    public RegionEngine getRegionEngine(final long regionId) {
+        return this.regionEngineTable.get(regionId);
+    }
+
+    public List getAllRegionEngines() {
+        return Lists.newArrayList(this.regionEngineTable.values());
+    }
+
+    public ExecutorService getReadIndexExecutor() {
+        return readIndexExecutor;
+    }
+
+    public void setReadIndexExecutor(ExecutorService readIndexExecutor) {
+        this.readIndexExecutor = readIndexExecutor;
+    }
+
+    public ExecutorService getRaftStateTrigger() {
+        return raftStateTrigger;
+    }
+
+    public void setRaftStateTrigger(ExecutorService raftStateTrigger) {
+        this.raftStateTrigger = raftStateTrigger;
+    }
+
+    public ExecutorService getSnapshotExecutor() {
+        return snapshotExecutor;
+    }
+
+    public void setSnapshotExecutor(ExecutorService snapshotExecutor) {
+        this.snapshotExecutor = snapshotExecutor;
+    }
+
+    public ExecutorService getCliRpcExecutor() {
+        return cliRpcExecutor;
+    }
+
+    public void setCliRpcExecutor(ExecutorService cliRpcExecutor) {
+        this.cliRpcExecutor = cliRpcExecutor;
+    }
+
+    public ExecutorService getRaftRpcExecutor() {
+        return raftRpcExecutor;
+    }
+
+    public void setRaftRpcExecutor(ExecutorService raftRpcExecutor) {
+        this.raftRpcExecutor = raftRpcExecutor;
+    }
+
+    public ExecutorService getKvRpcExecutor() {
+        return kvRpcExecutor;
+    }
+
+    public void setKvRpcExecutor(ExecutorService kvRpcExecutor) {
+        this.kvRpcExecutor = kvRpcExecutor;
+    }
+
+    public ScheduledExecutorService getMetricsScheduler() {
+        return metricsScheduler;
+    }
+
+    public void setMetricsScheduler(ScheduledExecutorService metricsScheduler) {
+        this.metricsScheduler = metricsScheduler;
+    }
+
+    public ScheduledReporter getKvMetricsReporter() {
+        return kvMetricsReporter;
+    }
+
+    public void setKvMetricsReporter(ScheduledReporter kvMetricsReporter) {
+        this.kvMetricsReporter = kvMetricsReporter;
+    }
+
+    public ScheduledReporter getThreadPoolMetricsReporter() {
+        return threadPoolMetricsReporter;
+    }
+
+    public void setThreadPoolMetricsReporter(ScheduledReporter threadPoolMetricsReporter) {
+        this.threadPoolMetricsReporter = threadPoolMetricsReporter;
+    }
+
+    public boolean removeAndStopRegionEngine(final long regionId) {
+        final RegionEngine engine = this.regionEngineTable.get(regionId);
+        if (engine != null) {
+            engine.shutdown();
+            return true;
+        }
+        return false;
+    }
+
+    public StateListenerContainer getStateListenerContainer() {
+        return stateListenerContainer;
+    }
+
+    public List getLeaderRegionIds() {
+        final List regionIds = Lists.newArrayListWithCapacity(this.regionEngineTable.size());
+        for (final RegionEngine regionEngine : this.regionEngineTable.values()) {
+            if (regionEngine.isLeader()) {
+                regionIds.add(regionEngine.getRegion().getId());
+            }
+        }
+        return regionIds;
+    }
+
+    public int getRegionCount() {
+        return this.regionEngineTable.size();
+    }
+
+    public int getLeaderRegionCount() {
+        int count = 0;
+        for (final RegionEngine regionEngine : this.regionEngineTable.values()) {
+            if (regionEngine.isLeader()) {
+                count++;
+            }
+        }
+        return count;
+    }
+
+    public boolean isBusy() {
+        // Need more info
+        return splitting.get();
+    }
+
+    public void applySplit(final Long regionId, final Long newRegionId, final KVStoreClosure closure) {
+        Requires.requireNonNull(regionId, "regionId");
+        Requires.requireNonNull(newRegionId, "newRegionId");
+        if (this.regionEngineTable.containsKey(newRegionId)) {
+            closure.setError(Errors.CONFLICT_REGION_ID);
+            closure.run(new Status(-1, "Conflict region id %d", newRegionId));
+            return;
+        }
+        if (!this.splitting.compareAndSet(false, true)) {
+            closure.setError(Errors.SERVER_BUSY);
+            closure.run(new Status(-1, "Server is busy now"));
+            return;
+        }
+        final RegionEngine parentEngine = getRegionEngine(regionId);
+        if (parentEngine == null) {
+            closure.setError(Errors.NO_REGION_FOUND);
+            closure.run(new Status(-1, "RegionEngine[%s] not found", regionId));
+            this.splitting.set(false);
+            return;
+        }
+        if (!parentEngine.isLeader()) {
+            closure.setError(Errors.NOT_LEADER);
+            closure.run(new Status(-1, "RegionEngine[%s] not leader", regionId));
+            this.splitting.set(false);
+            return;
+        }
+        final Region parentRegion = parentEngine.getRegion();
+        final byte[] startKey = BytesUtil.nullToEmpty(parentRegion.getStartKey());
+        final byte[] endKey = parentRegion.getEndKey();
+        final long approximateKeys = this.rawKVStore.getApproximateKeysInRange(startKey, endKey);
+        final long leastKeysOnSplit = this.storeOpts.getLeastKeysOnSplit();
+        if (approximateKeys < leastKeysOnSplit) {
+            closure.setError(Errors.TOO_SMALL_TO_SPLIT);
+            closure.run(new Status(-1, "RegionEngine[%s]'s keys less than %d", regionId, leastKeysOnSplit));
+            this.splitting.set(false);
+            return;
+        }
+        final byte[] splitKey = this.rawKVStore.jumpOver(startKey, approximateKeys >> 1);
+        if (splitKey == null) {
+            closure.setError(Errors.STORAGE_ERROR);
+            closure.run(new Status(-1, "Fail to scan split key"));
+            this.splitting.set(false);
+            return;
+        }
+        final KVOperation op = KVOperation.createRangeSplit(splitKey, regionId, newRegionId);
+        final Task task = new Task();
+        task.setData(ByteBuffer.wrap(Serializers.getDefault().writeObject(op)));
+        task.setDone(new KVClosureAdapter(closure, op));
+        parentEngine.getNode().apply(task);
+    }
+
+    public void doSplit(final Long regionId, final Long newRegionId, final byte[] splitKey) {
+        try {
+            Requires.requireNonNull(regionId, "regionId");
+            Requires.requireNonNull(newRegionId, "newRegionId");
+            final RegionEngine parent = getRegionEngine(regionId);
+            final Region region = parent.getRegion().copy();
+            final RegionEngineOptions rOpts = parent.copyRegionOpts();
+            region.setId(newRegionId);
+            region.setStartKey(splitKey);
+            region.setRegionEpoch(new RegionEpoch(-1, -1));
+
+            rOpts.setRegionId(newRegionId);
+            rOpts.setStartKeyBytes(region.getStartKey());
+            rOpts.setEndKeyBytes(region.getEndKey());
+            rOpts.setRaftGroupId(JRaftHelper.getJRaftGroupId(this.pdClient.getClusterName(), newRegionId));
+            rOpts.setRaftDataPath(null);
+
+            String baseRaftDataPath = this.storeOpts.getRaftDataPath();
+            if (Strings.isBlank(baseRaftDataPath)) {
+                baseRaftDataPath = "";
+            }
+            rOpts.setRaftDataPath(baseRaftDataPath + "raft_data_region_" + region.getId() + "_"
+                                  + getSelfEndpoint().getPort());
+            final RegionEngine engine = new RegionEngine(region, this);
+            if (!engine.init(rOpts)) {
+                LOG.error("Fail to init [RegionEngine: {}].", region);
+                throw Errors.REGION_ENGINE_FAIL.exception();
+            }
+
+            // update parent conf
+            final Region pRegion = parent.getRegion();
+            final RegionEpoch pEpoch = pRegion.getRegionEpoch();
+            final long version = pEpoch.getVersion();
+            pEpoch.setVersion(version + 1); // version + 1
+            pRegion.setEndKey(splitKey); // update endKey
+
+            // the following two lines of code can make a relation of 'happens-before' for
+            // read 'pRegion', because that a write to a ConcurrentMap happens-before every
+            // subsequent read of that ConcurrentMap.
+            this.regionEngineTable.put(region.getId(), engine);
+            registerRegionKVService(new DefaultRegionKVService(engine));
+
+            // update local regionRouteTable
+            this.pdClient.getRegionRouteTable().splitRegion(pRegion.getId(), region);
+        } finally {
+            this.splitting.set(false);
+        }
+    }
+
+    private void startMetricReporters(final long metricsReportPeriod) {
+        if (metricsReportPeriod <= 0) {
+            return;
+        }
+        if (this.kvMetricsReporter == null) {
+            if (this.metricsScheduler == null) {
+                // will sharing with all regionEngines
+                this.metricsScheduler = StoreEngineHelper.createMetricsScheduler();
+            }
+            // start kv store metrics reporter
+            this.kvMetricsReporter = Slf4jReporter.forRegistry(KVMetrics.metricRegistry()) //
+                .prefixedWith("store_" + this.storeId) //
+                .withLoggingLevel(Slf4jReporter.LoggingLevel.INFO) //
+                .outputTo(LOG) //
+                .scheduleOn(this.metricsScheduler) //
+                .shutdownExecutorOnStop(false) //
+                .build();
+            this.kvMetricsReporter.start(metricsReportPeriod, TimeUnit.SECONDS);
+        }
+        if (this.threadPoolMetricsReporter == null) {
+            if (this.metricsScheduler == null) {
+                // will sharing with all regionEngines
+                this.metricsScheduler = StoreEngineHelper.createMetricsScheduler();
+            }
+            // start threadPool metrics reporter
+            this.threadPoolMetricsReporter = Slf4jReporter.forRegistry(ThreadPoolMetricRegistry.metricRegistry()) //
+                .withLoggingLevel(Slf4jReporter.LoggingLevel.INFO) //
+                .outputTo(LOG) //
+                .scheduleOn(this.metricsScheduler) //
+                .shutdownExecutorOnStop(false) //
+                .build();
+            this.threadPoolMetricsReporter.start(metricsReportPeriod, TimeUnit.SECONDS);
+        }
+    }
+
+    private boolean initRawKVStore(final StoreEngineOptions opts) {
+        final StorageType storageType = opts.getStorageType();
+        switch (storageType) {
+            case RocksDB:
+                return initRocksDB(opts);
+            case Memory:
+                return initMemoryDB(opts);
+            default:
+                throw new UnsupportedOperationException("unsupported storage type: " + storageType);
+        }
+    }
+
+    private boolean initRocksDB(final StoreEngineOptions opts) {
+        RocksDBOptions rocksOpts = opts.getRocksDBOptions();
+        if (rocksOpts == null) {
+            rocksOpts = new RocksDBOptions();
+            opts.setRocksDBOptions(rocksOpts);
+        }
+        String dbPath = rocksOpts.getDbPath();
+        if (Strings.isNotBlank(dbPath)) {
+            try {
+                FileUtils.forceMkdir(new File(dbPath));
+            } catch (final Throwable t) {
+                LOG.error("Fail to make dir for dbPath {}.", dbPath);
+                return false;
+            }
+        } else {
+            dbPath = "";
+        }
+        final String childPath = "db_" + this.storeId + "_" + opts.getServerAddress().getPort();
+        rocksOpts.setDbPath(Paths.get(dbPath, childPath).toString());
+        this.dbPath = new File(rocksOpts.getDbPath());
+        final RocksRawKVStore rocksRawKVStore = new RocksRawKVStore();
+        if (!rocksRawKVStore.init(rocksOpts)) {
+            LOG.error("Fail to init [RocksRawKVStore].");
+            return false;
+        }
+        this.rawKVStore = rocksRawKVStore;
+        return true;
+    }
+
+    private boolean initMemoryDB(final StoreEngineOptions opts) {
+        MemoryDBOptions memoryOpts = opts.getMemoryDBOptions();
+        if (memoryOpts == null) {
+            memoryOpts = new MemoryDBOptions();
+            opts.setMemoryDBOptions(memoryOpts);
+        }
+        final MemoryRawKVStore memoryRawKVStore = new MemoryRawKVStore();
+        if (!memoryRawKVStore.init(memoryOpts)) {
+            LOG.error("Fail to init [MemoryRawKVStore].");
+            return false;
+        }
+        this.rawKVStore = memoryRawKVStore;
+        return true;
+    }
+
+    private boolean initAllRegionEngine(final StoreEngineOptions opts, final Store store) {
+        Requires.requireNonNull(opts, "opts");
+        Requires.requireNonNull(store, "store");
+        String baseRaftDataPath = opts.getRaftDataPath();
+        if (Strings.isNotBlank(baseRaftDataPath)) {
+            try {
+                FileUtils.forceMkdir(new File(baseRaftDataPath));
+            } catch (final Throwable t) {
+                LOG.error("Fail to make dir for raftDataPath: {}.", baseRaftDataPath);
+                return false;
+            }
+        } else {
+            baseRaftDataPath = "";
+        }
+        final Endpoint serverAddress = opts.getServerAddress();
+        final List rOptsList = opts.getRegionEngineOptionsList();
+        final List regionList = store.getRegions();
+        Requires.requireTrue(rOptsList.size() == regionList.size());
+        for (int i = 0; i < rOptsList.size(); i++) {
+            final RegionEngineOptions rOpts = rOptsList.get(i);
+            if (!inConfiguration(rOpts.getServerAddress().toString(), rOpts.getInitialServerList())) {
+                continue;
+            }
+            final Region region = regionList.get(i);
+            if (Strings.isBlank(rOpts.getRaftDataPath())) {
+                final String childPath = "raft_data_region_" + region.getId() + "_" + serverAddress.getPort();
+                rOpts.setRaftDataPath(Paths.get(baseRaftDataPath, childPath).toString());
+            }
+            Requires.requireNonNull(region.getRegionEpoch(), "regionEpoch");
+            final RegionEngine engine = new RegionEngine(region, this);
+            if (engine.init(rOpts)) {
+                final RegionKVService regionKVService = new DefaultRegionKVService(engine);
+                registerRegionKVService(regionKVService);
+                this.regionEngineTable.put(region.getId(), engine);
+            } else {
+                LOG.error("Fail to init [RegionEngine: {}].", region);
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean inConfiguration(final String curr, final String all) {
+        final PeerId currPeer = new PeerId();
+        if (!currPeer.parse(curr)) {
+            return false;
+        }
+        final Configuration allConf = new Configuration();
+        if (!allConf.parse(all)) {
+            return false;
+        }
+        return allConf.contains(currPeer) || allConf.getLearners().contains(currPeer);
+    }
+
+    private void registerRegionKVService(final RegionKVService regionKVService) {
+        final RegionKVService preService = this.regionKVServiceTable.putIfAbsent(regionKVService.getRegionId(),
+            regionKVService);
+        if (preService != null) {
+            throw new RheaRuntimeException("RegionKVService[region=" + regionKVService.getRegionId()
+                                           + "] has already been registered, can not register again!");
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "StoreEngine{storeId=" + storeId + ", startTime=" + startTime + ", dbPath=" + dbPath + ", storeOpts="
+               + storeOpts + ", started=" + started + ", regions=" + getAllRegionEngines() + '}';
+    }
+
+    @Override
+    public void describe(final Printer out) {
+        out.println("StoreEngine:"); //
+        out.print("  AllLeaderRegions:") //
+            .println(getLeaderRegionIds()); //
+        for (final RegionEngine r : getAllRegionEngines()) {
+            r.describe(out);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngineHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngineHelper.java
new file mode 100644
index 0000000..15ef09c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/StoreEngineHelper.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.SynchronousQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanRequest;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class StoreEngineHelper {
+
+    public static ExecutorService createReadIndexExecutor(final int coreThreads) {
+        final int maxThreads = coreThreads << 2;
+        final RejectedExecutionHandler handler = new ThreadPoolExecutor.AbortPolicy();
+        return newPool(coreThreads, maxThreads, "rheakv-read-index-callback", handler);
+    }
+
+    public static ExecutorService createRaftStateTrigger(final int coreThreads) {
+        final BlockingQueue workQueue = new ArrayBlockingQueue<>(32);
+        return newPool(coreThreads, coreThreads, "rheakv-raft-state-trigger", workQueue);
+    }
+
+    public static ExecutorService createSnapshotExecutor(final int coreThreads, final int maxThreads) {
+        return newPool(coreThreads, maxThreads, "rheakv-snapshot-executor");
+    }
+
+    public static ExecutorService createCliRpcExecutor(final int coreThreads) {
+        final int maxThreads = coreThreads << 2;
+        return newPool(coreThreads, maxThreads, "rheakv-cli-rpc-executor");
+    }
+
+    public static ExecutorService createRaftRpcExecutor(final int coreThreads) {
+        final int maxThreads = coreThreads << 1;
+        return newPool(coreThreads, maxThreads, "rheakv-raft-rpc-executor");
+    }
+
+    public static ExecutorService createKvRpcExecutor(final int coreThreads) {
+        final int maxThreads = coreThreads << 2;
+        return newPool(coreThreads, maxThreads, "rheakv-kv-store-rpc-executor");
+    }
+
+    public static ScheduledExecutorService createMetricsScheduler() {
+        return Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("rheakv-metrics-reporter", true));
+    }
+
+    public static void addKvStoreRequestProcessor(final RpcServer rpcServer, final StoreEngine engine) {
+        rpcServer.registerProcessor(new KVCommandProcessor<>(GetRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(MultiGetRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(ContainsKeyRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(GetSequenceRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(ResetSequenceRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(ScanRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(PutRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(GetAndPutRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(CompareAndPutRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(MergeRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(PutIfAbsentRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(KeyLockRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(KeyUnlockRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(BatchPutRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(DeleteRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(DeleteRangeRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(BatchDeleteRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(NodeExecuteRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(RangeSplitRequest.class, engine));
+        rpcServer.registerProcessor(new KVCommandProcessor<>(CASAllRequest.class, engine));
+    }
+
+    private static ExecutorService newPool(final int coreThreads, final int maxThreads, final String name) {
+        final RejectedExecutionHandler defaultHandler = new CallerRunsPolicyWithReport(name, name);
+        return newPool(coreThreads, maxThreads, name, defaultHandler);
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private static ExecutorService newPool(final int coreThreads, final int maxThreads, final String name,
+                                           final BlockingQueue workQueue) {
+        final RejectedExecutionHandler defaultHandler = new CallerRunsPolicyWithReport(name, name);
+        return newPool(coreThreads, maxThreads, workQueue, name, defaultHandler);
+    }
+
+    private static ExecutorService newPool(final int coreThreads, final int maxThreads, final String name,
+                                           final RejectedExecutionHandler handler) {
+        final BlockingQueue defaultWorkQueue = new SynchronousQueue<>();
+        return newPool(coreThreads, maxThreads, defaultWorkQueue, name, handler);
+    }
+
+    private static ExecutorService newPool(final int coreThreads, final int maxThreads,
+                                           final BlockingQueue workQueue, final String name,
+                                           final RejectedExecutionHandler handler) {
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(true) //
+            .coreThreads(coreThreads) //
+            .maximumThreads(maxThreads) //
+            .keepAliveSeconds(60L) //
+            .workQueue(workQueue) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(handler) //
+            .build();
+    }
+
+    private StoreEngineHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultDistributedLock.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultDistributedLock.java
new file mode 100644
index 0000000..7e07510
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultDistributedLock.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.errors.InvalidLockAcquirerException;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * Default implementation with distributed lock.
+ *
+ * @author jiachun.fjc
+ */
+class DefaultDistributedLock extends DistributedLock {
+
+    private static final Logger         LOG                = LoggerFactory.getLogger(DefaultDistributedLock.class);
+
+    private final DefaultRheaKVStore    rheaKVStore;
+
+    private volatile ScheduledFuture watchdogFuture;
+    private volatile boolean            mayCancelIfRunning = false;
+
+    protected DefaultDistributedLock(byte[] target, long lease, TimeUnit unit, ScheduledExecutorService watchdog,
+                                     DefaultRheaKVStore rheaKVStore) {
+        super(target, lease, unit, watchdog);
+        this.rheaKVStore = rheaKVStore;
+    }
+
+    @Override
+    public void unlock() {
+        final byte[] internalKey = getInternalKey();
+        final Acquirer acquirer = getAcquirer();
+        try {
+            final Owner owner = this.rheaKVStore.releaseLockWith(internalKey, acquirer).get();
+            updateOwner(owner);
+            if (!owner.isSameAcquirer(acquirer)) {
+                final String message = String.format(
+                    "an invalid acquirer [%s] trying to unlock, the real owner is [%s]", acquirer, owner);
+                throw new InvalidLockAcquirerException(message);
+            }
+            if (owner.getAcquires() <= 0) {
+                tryCancelScheduling();
+            }
+        } catch (final InvalidLockAcquirerException e) {
+            LOG.error("Fail to unlock, {}.", StackTraceUtil.stackTrace(e));
+            ThrowUtil.throwException(e);
+        } catch (final Throwable t) {
+            LOG.error("Fail to unlock: {}, will cancel scheduling, {}.", acquirer, StackTraceUtil.stackTrace(t));
+            tryCancelScheduling();
+            ThrowUtil.throwException(t);
+        }
+    }
+
+    @Override
+    protected Owner internalTryLock(final byte[] ctx) {
+        final byte[] internalKey = getInternalKey();
+        final Acquirer acquirer = getAcquirer();
+        acquirer.setContext(ctx);
+        final CompletableFuture future = this.rheaKVStore.tryLockWith(internalKey, false, acquirer);
+        final Owner owner = FutureHelper.get(future);
+        if (!owner.isSuccess()) {
+            updateOwner(owner);
+            return owner;
+        }
+
+        // if success, update the fencing token in acquirer
+        updateOwnerAndAcquirer(owner);
+
+        final ScheduledExecutorService watchdog = getWatchdog();
+        if (watchdog == null) {
+            return owner;
+        }
+        // schedule keeping lease
+        if (this.watchdogFuture == null) {
+            synchronized (this) {
+                if (this.watchdogFuture == null) {
+                    final long period = (acquirer.getLeaseMillis() / 3) << 1;
+                    this.watchdogFuture = scheduleKeepingLease(watchdog, internalKey, acquirer, period);
+                }
+            }
+        }
+        return owner;
+    }
+
+    private ScheduledFuture scheduleKeepingLease(final ScheduledExecutorService watchdog, final byte[] key,
+                                                    final Acquirer acquirer, final long period) {
+        return watchdog.scheduleAtFixedRate(() -> {
+            try {
+                if (this.mayCancelIfRunning) {
+                    // last time fail to cancel
+                    tryCancelScheduling();
+                    return;
+                }
+                this.rheaKVStore.tryLockWith(key, true, acquirer).whenComplete((result, throwable) -> {
+                    if (throwable != null) {
+                        LOG.error("Fail to keeping lease with lock: {}, {}.", acquirer, StackTraceUtil.stackTrace(throwable));
+                        tryCancelScheduling();
+                        return;
+                    }
+                    if (!result.isSuccess()) {
+                        LOG.warn("Fail to keeping lease with lock: {}, and result detail is: {}.", acquirer, result);
+                        tryCancelScheduling();
+                        return;
+                    }
+                    LOG.debug("Keeping lease with lock: {}.", acquirer);
+                });
+            } catch (final Throwable t) {
+                LOG.error("Fail to keeping lease with lock: {}, {}.", acquirer, StackTraceUtil.stackTrace(t));
+                tryCancelScheduling();
+            }
+        }, period, period, TimeUnit.MILLISECONDS);
+    }
+
+    private void tryCancelScheduling() {
+        if (this.watchdogFuture != null) {
+            this.mayCancelIfRunning = true;
+            this.watchdogFuture.cancel(true);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaIterator.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaIterator.java
new file mode 100644
index 0000000..e2b3b46
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaIterator.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.ArrayDeque;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultRheaIterator implements RheaIterator {
+
+    private final DefaultRheaKVStore    rheaKVStore;
+    private final PlacementDriverClient pdClient;
+    private final byte[]                startKey;
+    private final byte[]                endKey;
+    private final boolean               readOnlySafe;
+    private final boolean               returnValue;
+    private final int                   bufSize;
+    private final Queue        buf;
+
+    private byte[]                      cursorKey;
+
+    public DefaultRheaIterator(DefaultRheaKVStore rheaKVStore, byte[] startKey, byte[] endKey, int bufSize,
+                               boolean readOnlySafe, boolean returnValue) {
+        this.rheaKVStore = rheaKVStore;
+        this.pdClient = rheaKVStore.getPlacementDriverClient();
+        this.startKey = BytesUtil.nullToEmpty(startKey);
+        this.endKey = endKey;
+        this.bufSize = bufSize;
+        this.readOnlySafe = readOnlySafe;
+        this.returnValue = returnValue;
+        this.buf = new ArrayDeque<>(bufSize);
+        this.cursorKey = this.startKey;
+    }
+
+    @Override
+    public synchronized boolean hasNext() {
+        if (this.buf.isEmpty()) {
+            while (this.endKey == null || BytesUtil.compare(this.cursorKey, this.endKey) < 0) {
+                final List kvEntries = this.rheaKVStore.singleRegionScan(this.cursorKey, this.endKey,
+                    this.bufSize, this.readOnlySafe, this.returnValue);
+                if (kvEntries.isEmpty()) {
+                    // cursorKey jump to next region's startKey
+                    this.cursorKey = this.pdClient.findStartKeyOfNextRegion(this.cursorKey, false);
+                    if (cursorKey == null) { // current is the last region
+                        break;
+                    }
+                } else {
+                    final KVEntry last = kvEntries.get(kvEntries.size() - 1);
+                    this.cursorKey = BytesUtil.nextBytes(last.getKey()); // cursorKey++
+                    this.buf.addAll(kvEntries);
+                    break;
+                }
+            }
+            return !this.buf.isEmpty();
+        }
+        return true;
+    }
+
+    @Override
+    public synchronized KVEntry next() {
+        if (this.buf.isEmpty()) {
+            throw new NoSuchElementException();
+        }
+        return this.buf.poll();
+    }
+
+    public byte[] getStartKey() {
+        return startKey;
+    }
+
+    public byte[] getEndKey() {
+        return endKey;
+    }
+
+    public boolean isReadOnlySafe() {
+        return readOnlySafe;
+    }
+
+    public int getBufSize() {
+        return bufSize;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVCliService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVCliService.java
new file mode 100644
index 0000000..096cb6c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVCliService.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.CliService;
+import com.alipay.sofa.jraft.RaftServiceFactory;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.CliServiceImpl;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.store.RangeSplitRequest;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rpc.CliClientService;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.AbstractClientService;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultRheaKVCliService implements RheaKVCliService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultRheaKVCliService.class);
+
+    private RpcClient           rpcClient;
+    private CliService          cliService;
+    private CliOptions          opts;
+
+    private boolean             started;
+
+    @Override
+    public boolean init(final CliOptions opts) {
+        if (this.started) {
+            LOG.info("[DefaultRheaKVRpcService] already started.");
+            return true;
+        }
+        initCli(opts);
+        LOG.info("[DefaultRheaKVCliService] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public void shutdown() {
+        if (this.cliService != null) {
+            this.cliService.shutdown();
+        }
+        this.started = false;
+        LOG.info("[DefaultRheaKVCliService] shutdown successfully.");
+    }
+
+    @Override
+    public Status rangeSplit(final long regionId, final long newRegionId, final String groupId, final Configuration conf) {
+        final PeerId leaderId = new PeerId();
+        final Status st = this.cliService.getLeader(groupId, conf, leaderId);
+        if (!st.isOk()) {
+            throw new IllegalStateException(st.getErrorMsg());
+        }
+        final RangeSplitRequest request = new RangeSplitRequest();
+        request.setRegionId(regionId);
+        request.setNewRegionId(newRegionId);
+        try {
+            final BaseResponse response = (BaseResponse) this.rpcClient.invokeSync(leaderId.getEndpoint(),
+                request, this.opts.getTimeoutMs());
+            if (response.isSuccess()) {
+                return Status.OK();
+            }
+            return new Status(-1, "Fail to range split on region %d, error: %s", regionId, response);
+        } catch (final Exception e) {
+            LOG.error("Fail to range split on exception: {}.", StackTraceUtil.stackTrace(e));
+            return new Status(-1, "fail to range split on region %d", regionId);
+        }
+    }
+
+    private void initCli(CliOptions cliOpts) {
+        if (cliOpts == null) {
+            cliOpts = new CliOptions();
+            cliOpts.setTimeoutMs(5000);
+            cliOpts.setMaxRetry(3);
+        }
+        this.opts = cliOpts;
+        this.cliService = RaftServiceFactory.createAndInitCliService(cliOpts);
+        final CliClientService cliClientService = ((CliServiceImpl) this.cliService).getCliClientService();
+        Requires.requireNonNull(cliClientService, "cliClientService");
+        this.rpcClient = ((AbstractClientService) cliClientService).getRpcClient();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVRpcService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVRpcService.java
new file mode 100644
index 0000000..6f5a8ef
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVRpcService.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.pd.AbstractPlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseResponse;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.ErrorsHelper;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.rpc.ExtSerializerSupports;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.InvokeContext;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultRheaKVRpcService implements RheaKVRpcService {
+
+    private static final Logger         LOG = LoggerFactory.getLogger(DefaultRheaKVRpcService.class);
+
+    private final PlacementDriverClient pdClient;
+    private final RpcClient             rpcClient;
+    private final Endpoint              selfEndpoint;
+
+    private ThreadPoolExecutor          rpcCallbackExecutor;
+    private int                         rpcTimeoutMillis;
+
+    private boolean                     started;
+
+    public DefaultRheaKVRpcService(PlacementDriverClient pdClient, Endpoint selfEndpoint) {
+        this.pdClient = pdClient;
+        this.rpcClient = ((AbstractPlacementDriverClient) pdClient).getRpcClient();
+        this.selfEndpoint = selfEndpoint;
+    }
+
+    @Override
+    public synchronized boolean init(final RpcOptions opts) {
+        if (this.started) {
+            LOG.info("[DefaultRheaKVRpcService] already started.");
+            return true;
+        }
+        this.rpcCallbackExecutor = createRpcCallbackExecutor(opts);
+        this.rpcTimeoutMillis = opts.getRpcTimeoutMillis();
+        Requires.requireTrue(this.rpcTimeoutMillis > 0, "opts.rpcTimeoutMillis must > 0");
+        LOG.info("[DefaultRheaKVRpcService] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.rpcCallbackExecutor);
+        this.started = false;
+        LOG.info("[DefaultRheaKVRpcService] shutdown successfully.");
+    }
+
+    @Override
+    public  CompletableFuture callAsyncWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                                     final Errors lastCause) {
+        return callAsyncWithRpc(request, closure, lastCause, true);
+    }
+
+    @Override
+    public  CompletableFuture callAsyncWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                                     final Errors lastCause, final boolean requireLeader) {
+        final boolean forceRefresh = ErrorsHelper.isInvalidPeer(lastCause);
+        final Endpoint endpoint = getRpcEndpoint(request.getRegionId(), forceRefresh, this.rpcTimeoutMillis,
+            requireLeader);
+        internalCallAsyncWithRpc(endpoint, request, closure);
+        return closure.future();
+    }
+
+    public Endpoint getLeader(final long regionId, final boolean forceRefresh, final long timeoutMillis) {
+        return this.pdClient.getLeader(regionId, forceRefresh, timeoutMillis);
+    }
+
+    public Endpoint getLuckyPeer(final long regionId, final boolean forceRefresh, final long timeoutMillis) {
+        return this.pdClient.getLuckyPeer(regionId, forceRefresh, timeoutMillis, this.selfEndpoint);
+    }
+
+    public Endpoint getRpcEndpoint(final long regionId, final boolean forceRefresh, final long timeoutMillis,
+                                   final boolean requireLeader) {
+        if (requireLeader) {
+            return getLeader(regionId, forceRefresh, timeoutMillis);
+        } else {
+            return getLuckyPeer(regionId, forceRefresh, timeoutMillis);
+        }
+    }
+
+    private  void internalCallAsyncWithRpc(final Endpoint endpoint, final BaseRequest request,
+                                              final FailoverClosure closure) {
+        final InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(BoltRpcClient.BOLT_CTX, ExtSerializerSupports.getInvokeContext());
+        final InvokeCallback invokeCallback = new InvokeCallback() {
+
+            @Override
+            public void complete(final Object result, final Throwable err) {
+                if (err == null) {
+                    final BaseResponse response = (BaseResponse) result;
+                    if (response.isSuccess()) {
+                        closure.setData(response.getValue());
+                        closure.run(Status.OK());
+                    } else {
+                        closure.setError(response.getError());
+                        closure.run(new Status(-1, "RPC failed with address: %s, response: %s", endpoint, response));
+                    }
+                } else {
+                    closure.failure(err);
+                }
+            }
+
+            @Override
+            public Executor executor() {
+                return rpcCallbackExecutor;
+            }
+        };
+
+        try {
+            this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback, this.rpcTimeoutMillis);
+        } catch (final Throwable t) {
+            closure.failure(t);
+        }
+    }
+
+    private ThreadPoolExecutor createRpcCallbackExecutor(final RpcOptions opts) {
+        final int callbackExecutorCorePoolSize = opts.getCallbackExecutorCorePoolSize();
+        final int callbackExecutorMaximumPoolSize = opts.getCallbackExecutorMaximumPoolSize();
+        if (callbackExecutorCorePoolSize <= 0 || callbackExecutorMaximumPoolSize <= 0) {
+            return null;
+        }
+
+        final String name = "rheakv-rpc-callback";
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(true) //
+            .coreThreads(callbackExecutorCorePoolSize) //
+            .maximumThreads(callbackExecutorMaximumPoolSize) //
+            .keepAliveSeconds(120L) //
+            .workQueue(new ArrayBlockingQueue<>(opts.getCallbackExecutorQueueCapacity())) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(new CallerRunsPolicyWithReport(name)) //
+            .build();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVStore.java
new file mode 100644
index 0000000..783bdcf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/DefaultRheaKVStore.java
@@ -0,0 +1,1939 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.rhea.storage.zip.ZipStrategyManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.DescriberManager;
+import com.alipay.sofa.jraft.rhea.FollowerStateListener;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.LeaderStateListener;
+import com.alipay.sofa.jraft.rhea.RegionEngine;
+import com.alipay.sofa.jraft.rhea.StateListener;
+import com.alipay.sofa.jraft.rhea.StateListenerContainer;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.failover.ListRetryCallable;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryCallable;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryRunner;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.BoolFailoverFuture;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.FailoverClosureImpl;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.ListFailoverFuture;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.MapFailoverFuture;
+import com.alipay.sofa.jraft.rhea.client.pd.FakePlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.client.pd.RemotePlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.cmd.store.CASAllRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchDeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.BatchPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.CompareAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ContainsKeyRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRangeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.DeleteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetAndPutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.GetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyLockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.KeyUnlockRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MergeRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.MultiGetRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.NodeExecuteRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutIfAbsentRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.PutRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ResetSequenceRequest;
+import com.alipay.sofa.jraft.rhea.cmd.store.ScanRequest;
+import com.alipay.sofa.jraft.rhea.errors.ApiExceptionHelper;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.ErrorsHelper;
+import com.alipay.sofa.jraft.rhea.errors.RheaRuntimeException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetricNames;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.options.BatchingOptions;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.rpc.ExtSerializerSupports;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVIterator;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.NodeExecutor;
+import com.alipay.sofa.jraft.rhea.storage.RawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Constants;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.rhea.util.concurrent.AffinityNamedThreadFactory;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rhea.util.concurrent.disruptor.Dispatcher;
+import com.alipay.sofa.jraft.rhea.util.concurrent.disruptor.TaskDispatcher;
+import com.alipay.sofa.jraft.rhea.util.concurrent.disruptor.WaitStrategyType;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.LogExceptionHandler;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+import com.codahale.metrics.Histogram;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.dsl.Disruptor;
+
+/**
+ * Default client of RheaKV store implementation.
+ *
+ * For example, the processing flow of the method {@link #scan(byte[], byte[])},
+ * and the implementation principle of failover:
+ *
+ * 
+ * 1. The first step is to filter out region1, region2, and region3 from the routing table.
+ *
+ *                ┌─────────────────┐                               ┌─────────────────┐
+ *                │  scan startKey  │                               │   scan endKey   │
+ *                └────────┬────────┘                               └────────┬────────┘
+ *                         │                                                 │
+ *                         │                                                 │
+ *                         │                                                 │
+ * ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─     │  ┌ ─ ─ ─ ─ ─ ─ ┐           ┌ ─ ─ ─ ─ ─ ─ ┐      │    ┌ ─ ─ ─ ─ ─ ─ ┐
+ *  startKey1=byte[0] │    │     startKey2                 startKey3         │       startKey4
+ * └ ─ ─ ─ ┬ ─ ─ ─ ─ ─     │  └ ─ ─ ─│─ ─ ─ ┘           └ ─ ─ ─│─ ─ ─ ┘      │    └ ─ ─ ─│─ ─ ─ ┘
+ *         │               │         │                         │             │           │
+ *         ▼───────────────▼─────────▼─────────────────────────▼─────────────▼───────────▼─────────────────────────┐
+ *         │                         │                         │                         │                         │
+ *         │                         │                         │                         │                         │
+ *         │         region1         │         region2         │          region3        │         region4         │
+ *         │                         │                         │                         │                         │
+ *         └─────────────────────────┴─────────────────────────┴─────────────────────────┴─────────────────────────┘
+ *
+ * 2. The second step is to split the request(scan -> multi-region scan):
+ *          region1->regionScan(startKey, regionEndKey1)
+ *          region2->regionScan(regionStartKey2, regionEndKey2)
+ *          region3->regionScan(regionStartKey3, endKey)
+ *
+ *            ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─     ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─      ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ *                call region1   │        call region2   │         call region3   │
+ *            └ ─ ─ ─ ─ ─ ─ ─ ─ ─     └ ─ ─ ─ ─ ─ ─ ─ ─ ─      └ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ *                     ║                       ║                        ║
+ *
+ *                     ║                       ║                        ║
+ *                     ▽                       ▽                        ▽
+ *     ┌─────────────────────────┬─────────────────────────┬─────────────────────────┬─────────────────────────┐
+ *     │                         │                         │                         │                         │
+ *     │                         │                         │                         │                         │
+ *     │         region1         │         region2         │          region3        │         region4         │
+ *     │                         │                         │                         │                         │
+ *     └─────────────────────────┴─────────────────────────┴─────────────────────────┴─────────────────────────┘
+ *
+ * 3. The third step, encountering the region split (the sign of the split is the change of the region epoch)
+ *      To refresh the RegionRouteTable, you need to obtain the latest routing table from the PD.
+ *
+ *      For example, region2 is split into region2 + region5:
+ *          The request 'region2->regionScan(regionStartKey2, regionEndKey2)' split and retry
+ *              1. region2->regionScan(regionStartKey2, newRegionEndKey2)
+ *              2. region5->regionScan(regionStartKey5, regionEndKey5)
+ *
+ *            ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─                              ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ *                call region1   │                                 call region3   │
+ *            └ ─ ─ ─ ─ ─ ─ ─ ─ ─                              └ ─ ─ ─ ─ ─ ─ ─ ─ ─
+ *                     ║        ┌ ─ ─ ─ ─ ─ ─ ┐                         ║
+ *                               retry region2
+ *                     ║        └ ─ ─ ─ ─ ─ ─ ┘┌ ─ ─ ─ ─ ─ ─ ┐          ║
+ *                                     ║        retry region5
+ *                     ║                       └ ─ ─ ─ ─ ─ ─ ┘          ║
+ *                                     ║              ║
+ *                     ║                                                ║
+ *                     ▽               ▽              ▽                 ▽
+ *     ┌─────────────────────────┬────────────┬ ─ ─ ─ ─ ─ ─┌─────────────────────────┬─────────────────────────┐
+ *     │                         │            │            │                         │                         │
+ *     │                         │            │            │                         │                         │
+ *     │         region1         │  region2   │  region5   │          region3        │         region4         │
+ *     │                         │            │            │                         │                         │
+ *     └─────────────────────────┴────────────┘─ ─ ─ ─ ─ ─ ┴─────────────────────────┴─────────────────────────┘
+ *
+ * 4. Encountering 'Invalid-Peer'(NOT_LEADER, NO_REGION_FOUND, LEADER_NOT_AVAILABLE)
+ *      This is very simple, re-acquire the latest leader of the raft-group to which the current key belongs,
+ *      and then call again.
+ * 

+ *
+ * @author jiachun.fjc
+ */
+public class DefaultRheaKVStore implements RheaKVStore {
+
+    private static final Logger                LOG                    = LoggerFactory
+                                                                          .getLogger(DefaultRheaKVStore.class);
+
+    static {
+        ExtSerializerSupports.init();
+    }
+
+    private final StateListenerContainer stateListenerContainer = new StateListenerContainer<>();
+    private StoreEngine                        storeEngine;
+    private PlacementDriverClient              pdClient;
+    private RheaKVRpcService                   rheaKVRpcService;
+    private RheaKVStoreOptions                 opts;
+    private int                                failoverRetries;
+    private long                               futureTimeoutMillis;
+    private boolean                            onlyLeaderRead;
+    private Dispatcher                         kvDispatcher;
+    private BatchingOptions                    batchingOpts;
+    private GetBatching                        getBatching;
+    private GetBatching                        getBatchingOnlySafe;
+    private PutBatching                        putBatching;
+
+    private volatile boolean                   started;
+
+    @Override
+    public synchronized boolean init(final RheaKVStoreOptions opts) {
+        if (this.started) {
+            LOG.info("[DefaultRheaKVStore] already started.");
+            return true;
+        }
+
+        DescriberManager.getInstance().addDescriber(RouteTable.getInstance());
+
+        this.opts = opts;
+        // init placement driver
+        final PlacementDriverOptions pdOpts = opts.getPlacementDriverOptions();
+        final String clusterName = opts.getClusterName();
+        Requires.requireNonNull(pdOpts, "opts.placementDriverOptions");
+        Requires.requireNonNull(clusterName, "opts.clusterName");
+        if (Strings.isBlank(pdOpts.getInitialServerList())) {
+            // if blank, extends parent's value
+            pdOpts.setInitialServerList(opts.getInitialServerList());
+        }
+        if (pdOpts.isFake()) {
+            this.pdClient = new FakePlacementDriverClient(opts.getClusterId(), clusterName);
+        } else {
+            this.pdClient = new RemotePlacementDriverClient(opts.getClusterId(), clusterName);
+        }
+        if (!this.pdClient.init(pdOpts)) {
+            LOG.error("Fail to init [PlacementDriverClient].");
+            return false;
+        }
+        // init compress strategies
+        ZipStrategyManager.init(opts);
+        // init store engine
+        final StoreEngineOptions stOpts = opts.getStoreEngineOptions();
+        if (stOpts != null) {
+            stOpts.setInitialServerList(opts.getInitialServerList());
+            this.storeEngine = new StoreEngine(this.pdClient, this.stateListenerContainer);
+            if (!this.storeEngine.init(stOpts)) {
+                LOG.error("Fail to init [StoreEngine].");
+                return false;
+            }
+        }
+        final Endpoint selfEndpoint = this.storeEngine == null ? null : this.storeEngine.getSelfEndpoint();
+        final RpcOptions rpcOpts = opts.getRpcOptions();
+        Requires.requireNonNull(rpcOpts, "opts.rpcOptions");
+        this.rheaKVRpcService = new DefaultRheaKVRpcService(this.pdClient, selfEndpoint) {
+
+            @Override
+            public Endpoint getLeader(final long regionId, final boolean forceRefresh, final long timeoutMillis) {
+                final Endpoint leader = getLeaderByRegionEngine(regionId);
+                if (leader != null) {
+                    return leader;
+                }
+                return super.getLeader(regionId, forceRefresh, timeoutMillis);
+            }
+        };
+        if (!this.rheaKVRpcService.init(rpcOpts)) {
+            LOG.error("Fail to init [RheaKVRpcService].");
+            return false;
+        }
+        this.failoverRetries = opts.getFailoverRetries();
+        this.futureTimeoutMillis = opts.getFutureTimeoutMillis();
+        this.onlyLeaderRead = opts.isOnlyLeaderRead();
+        if (opts.isUseParallelKVExecutor()) {
+            final int numWorkers = Utils.cpus();
+            final int bufSize = numWorkers << 4;
+            final String name = "parallel-kv-executor";
+            final ThreadFactory threadFactory = Constants.THREAD_AFFINITY_ENABLED
+                    ? new AffinityNamedThreadFactory(name, true) : new NamedThreadFactory(name, true);
+            this.kvDispatcher = new TaskDispatcher(bufSize, numWorkers, WaitStrategyType.LITE_BLOCKING_WAIT, threadFactory);
+        }
+        this.batchingOpts = opts.getBatchingOptions();
+        if (this.batchingOpts.isAllowBatching()) {
+            this.getBatching = new GetBatching(KeyEvent::new, "get_batching",
+                    new GetBatchingHandler("get", false));
+            this.getBatchingOnlySafe = new GetBatching(KeyEvent::new, "get_batching_only_safe",
+                    new GetBatchingHandler("get_only_safe", true));
+            this.putBatching = new PutBatching(KVEvent::new, "put_batching",
+                    new PutBatchingHandler("put"));
+        }
+        LOG.info("[DefaultRheaKVStore] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        this.started = false;
+        if (this.pdClient != null) {
+            this.pdClient.shutdown();
+        }
+        if (this.storeEngine != null) {
+            this.storeEngine.shutdown();
+        }
+        if (this.rheaKVRpcService != null) {
+            this.rheaKVRpcService.shutdown();
+        }
+        if (this.kvDispatcher != null) {
+            this.kvDispatcher.shutdown();
+        }
+        if (this.getBatching != null) {
+            this.getBatching.shutdown();
+        }
+        if (this.getBatchingOnlySafe != null) {
+            this.getBatchingOnlySafe.shutdown();
+        }
+        if (this.putBatching != null) {
+            this.putBatching.shutdown();
+        }
+        this.stateListenerContainer.clear();
+        LOG.info("[DefaultRheaKVStore] shutdown successfully.");
+    }
+
+    /**
+     * Returns a heap-allocated iterator over the contents of the
+     * database.
+     * 

+     * Caller should close the iterator when it is no longer needed.
+     * The returned iterator should be closed before this db is closed.
+     * 

+     * 
+     *     KVIterator it = unsafeLocalIterator();
+     *     try {
+     *         // do something
+     *     } finally {
+     *         it.close();
+     *     }
+     * 
+     */
+    public KVIterator unsafeLocalIterator() {
+        checkState();
+        if (this.pdClient instanceof RemotePlacementDriverClient) {
+            throw new UnsupportedOperationException("unsupported operation on multi-region");
+        }
+        if (this.storeEngine == null) {
+            throw new IllegalStateException("current node do not have store engine");
+        }
+        return this.storeEngine.getRawKVStore().localIterator();
+    }
+
+    @Override
+    public CompletableFuture get(final byte[] key) {
+        return get(key, true);
+    }
+
+    @Override
+    public CompletableFuture get(final String key) {
+        return get(BytesUtil.writeUtf8(key));
+    }
+
+    @Override
+    public CompletableFuture get(final byte[] key, final boolean readOnlySafe) {
+        Requires.requireNonNull(key, "key");
+        return get(key, readOnlySafe, new CompletableFuture<>(), true);
+    }
+
+    @Override
+    public CompletableFuture get(final String key, final boolean readOnlySafe) {
+        return get(BytesUtil.writeUtf8(key), readOnlySafe);
+    }
+
+    @Override
+    public byte[] bGet(final byte[] key) {
+        return FutureHelper.get(get(key), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public byte[] bGet(final String key) {
+        return FutureHelper.get(get(key), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public byte[] bGet(final byte[] key, final boolean readOnlySafe) {
+        return FutureHelper.get(get(key, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public byte[] bGet(final String key, final boolean readOnlySafe) {
+        return FutureHelper.get(get(key, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    private CompletableFuture get(final byte[] key, final boolean readOnlySafe,
+                                          final CompletableFuture future, final boolean tryBatching) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        if (tryBatching) {
+            final GetBatching getBatching = readOnlySafe ? this.getBatchingOnlySafe : this.getBatching;
+            if (getBatching != null && getBatching.apply(key, future)) {
+                return future;
+            }
+        }
+        internalGet(key, readOnlySafe, future, this.failoverRetries, null, this.onlyLeaderRead);
+        return future;
+    }
+
+    private void internalGet(final byte[] key, final boolean readOnlySafe, final CompletableFuture future,
+                             final int retriesLeft, final Errors lastCause, final boolean requireLeader) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), requireLeader);
+        // require leader on retry
+        final RetryRunner retryRunner = retryCause -> internalGet(key, readOnlySafe, future, retriesLeft - 1,
+                retryCause, true);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).get(key, readOnlySafe, closure);
+            }
+        } else {
+            final GetRequest request = new GetRequest();
+            request.setKey(key);
+            request.setReadOnlySafe(readOnlySafe);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause, requireLeader);
+        }
+    }
+
+    @Override
+    public CompletableFuture> multiGet(final List keys) {
+        return multiGet(keys, true);
+    }
+
+    @Override
+    public CompletableFuture> multiGet(final List keys, final boolean readOnlySafe) {
+        checkState();
+        Requires.requireNonNull(keys, "keys");
+        final FutureGroup> futureGroup = internalMultiGet(keys, readOnlySafe,
+            this.failoverRetries, null);
+        return FutureHelper.joinMap(futureGroup, keys.size());
+    }
+
+    @Override
+    public Map bMultiGet(final List keys) {
+        return FutureHelper.get(multiGet(keys), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Map bMultiGet(final List keys, final boolean readOnlySafe) {
+        return FutureHelper.get(multiGet(keys, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup> internalMultiGet(final List keys, final boolean readOnlySafe,
+                                                                 final int retriesLeft, final Throwable lastCause) {
+        final Map> regionMap = this.pdClient
+                .findRegionsByKeys(keys, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List>> futures = Lists.newArrayListWithCapacity(regionMap.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Map.Entry> entry : regionMap.entrySet()) {
+            final Region region = entry.getKey();
+            final List subKeys = entry.getValue();
+            final RetryCallable> retryCallable = retryCause -> internalMultiGet(subKeys,
+                    readOnlySafe, retriesLeft - 1, retryCause);
+            final MapFailoverFuture future = new MapFailoverFuture<>(retriesLeft, retryCallable);
+            internalRegionMultiGet(region, subKeys, readOnlySafe, future, retriesLeft, lastError, this.onlyLeaderRead);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    private void internalRegionMultiGet(final Region region, final List subKeys, final boolean readOnlySafe,
+                                        final CompletableFuture> future, final int retriesLeft,
+                                        final Errors lastCause, final boolean requireLeader) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), requireLeader);
+        // require leader on retry
+        final RetryRunner retryRunner = retryCause -> internalRegionMultiGet(region, subKeys, readOnlySafe, future,
+                retriesLeft - 1, retryCause, true);
+        final FailoverClosure> closure = new FailoverClosureImpl<>(future,
+                false, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                final RawKVStore rawKVStore = getRawKVStore(regionEngine);
+                if (this.kvDispatcher == null) {
+                    rawKVStore.multiGet(subKeys, readOnlySafe, closure);
+                } else {
+                    this.kvDispatcher.execute(() -> rawKVStore.multiGet(subKeys, readOnlySafe, closure));
+                }
+            }
+        } else {
+            final MultiGetRequest request = new MultiGetRequest();
+            request.setKeys(subKeys);
+            request.setReadOnlySafe(readOnlySafe);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause, requireLeader);
+        }
+    }
+
+    @Override
+    public CompletableFuture containsKey(final byte[] key) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalContainsKey(key, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture containsKey(final String key) {
+        return containsKey(BytesUtil.writeUtf8(key));
+    }
+
+    @Override
+    public Boolean bContainsKey(final byte[] key) {
+        return FutureHelper.get(containsKey(key), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bContainsKey(final String key) {
+        return FutureHelper.get(containsKey(key), this.futureTimeoutMillis);
+    }
+
+    private void internalContainsKey(final byte[] key, final CompletableFuture future,
+                                     final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalContainsKey(key, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).containsKey(key, closure);
+            }
+        } else {
+            final ContainsKeyRequest request = new ContainsKeyRequest();
+            request.setKey(key);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture> scan(final byte[] startKey, final byte[] endKey) {
+        return scan(startKey, endKey, true);
+    }
+
+    @Override
+    public CompletableFuture> scan(final String startKey, final String endKey) {
+        return scan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey));
+    }
+
+    @Override
+    public CompletableFuture> scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe) {
+        return scan(startKey, endKey, readOnlySafe, true);
+    }
+
+    @Override
+    public CompletableFuture> scan(final String startKey, final String endKey, final boolean readOnlySafe) {
+        return scan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), readOnlySafe);
+    }
+
+    @Override
+    public CompletableFuture> scan(final byte[] startKey, final byte[] endKey,
+                                                 final boolean readOnlySafe, final boolean returnValue) {
+        checkState();
+        final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+        if (endKey != null) {
+            Requires.requireTrue(BytesUtil.compare(realStartKey, endKey) < 0, "startKey must < endKey");
+        }
+        final FutureGroup> futureGroup = internalScan(realStartKey, endKey, readOnlySafe, returnValue,
+            this.failoverRetries, null);
+        return FutureHelper.joinList(futureGroup);
+    }
+
+    @Override
+    public CompletableFuture> scan(final String startKey, final String endKey,
+                                                 final boolean readOnlySafe, final boolean returnValue) {
+        return scan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), readOnlySafe, returnValue);
+    }
+
+    @Override
+    public List bScan(final byte[] startKey, final byte[] endKey) {
+        return FutureHelper.get(scan(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bScan(final String startKey, final String endKey) {
+        return FutureHelper.get(scan(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe) {
+        return FutureHelper.get(scan(startKey, endKey, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bScan(final String startKey, final String endKey, final boolean readOnlySafe) {
+        return FutureHelper.get(scan(startKey, endKey, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                               final boolean returnValue) {
+        return FutureHelper.get(scan(startKey, endKey, readOnlySafe, returnValue), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bScan(final String startKey, final String endKey, final boolean readOnlySafe,
+                               final boolean returnValue) {
+        return FutureHelper.get(scan(startKey, endKey, readOnlySafe, returnValue), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup> internalScan(final byte[] startKey, final byte[] endKey,
+                                                    final boolean readOnlySafe, final boolean returnValue,
+                                                    final int retriesLeft, final Throwable lastCause) {
+        Requires.requireNonNull(startKey, "startKey");
+        final List regionList = this.pdClient.findRegionsByKeyRange(startKey, endKey, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List>> futures = Lists.newArrayListWithCapacity(regionList.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Region region : regionList) {
+            final byte[] regionStartKey = region.getStartKey();
+            final byte[] regionEndKey = region.getEndKey();
+            final byte[] subStartKey = regionStartKey == null ? startKey : BytesUtil.max(regionStartKey, startKey);
+            final byte[] subEndKey = regionEndKey == null ? endKey :
+                    (endKey == null ? regionEndKey : BytesUtil.min(regionEndKey, endKey));
+            final ListRetryCallable retryCallable = retryCause -> internalScan(subStartKey, subEndKey,
+                    readOnlySafe, returnValue, retriesLeft - 1, retryCause);
+            final ListFailoverFuture future = new ListFailoverFuture<>(retriesLeft, retryCallable);
+            internalRegionScan(region, subStartKey, subEndKey, false, readOnlySafe, returnValue, future, retriesLeft,
+                    lastError, this.onlyLeaderRead);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    private void internalRegionScan(final Region region, final byte[] subStartKey, final byte[] subEndKey,
+                                    final boolean reverse, final boolean readOnlySafe, final boolean returnValue,
+                                    final CompletableFuture> future, final int retriesLeft,
+                                    final Errors lastCause, final boolean requireLeader) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), requireLeader);
+        // require leader on retry
+        final RetryRunner retryRunner = retryCause -> internalRegionScan(region, subStartKey, subEndKey, reverse, readOnlySafe,
+                returnValue, future, retriesLeft - 1, retryCause, true);
+        final FailoverClosure> closure = new FailoverClosureImpl<>(future, false,
+                retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                final RawKVStore rawKVStore = getRawKVStore(regionEngine);
+                if (reverse) {
+                    if (this.kvDispatcher == null) {
+                        rawKVStore.reverseScan(subStartKey, subEndKey, readOnlySafe, returnValue, closure);
+                    } else {
+                        this.kvDispatcher.execute(
+                                () -> rawKVStore.reverseScan(subStartKey, subEndKey, readOnlySafe, returnValue, closure));
+                    }
+                } else {
+                    if (this.kvDispatcher == null) {
+                        rawKVStore.scan(subStartKey, subEndKey, readOnlySafe, returnValue, closure);
+                    } else {
+                        this.kvDispatcher.execute(
+                                () -> rawKVStore.scan(subStartKey, subEndKey, readOnlySafe, returnValue, closure));
+                    }
+                }
+            }
+        } else {
+            final ScanRequest request = new ScanRequest();
+            request.setStartKey(subStartKey);
+            request.setEndKey(subEndKey);
+            request.setReadOnlySafe(readOnlySafe);
+            request.setReturnValue(returnValue);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            request.setReverse(reverse);
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause, requireLeader);
+        }
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey) {
+        return reverseScan(startKey, endKey, true);
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final String startKey, final String endKey) {
+        return reverseScan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey));
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey,
+                                                        final boolean readOnlySafe) {
+        return reverseScan(startKey, endKey, readOnlySafe, true);
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final String startKey, final String endKey,
+                                                        final boolean readOnlySafe) {
+        return reverseScan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), readOnlySafe);
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey,
+                                                        final boolean readOnlySafe, final boolean returnValue) {
+        checkState();
+        final byte[] realEndKey = BytesUtil.nullToEmpty(endKey);
+        if (startKey != null) {
+            Requires.requireTrue(BytesUtil.compare(startKey, realEndKey) > 0, "startKey must > endKey");
+        }
+        final FutureGroup> futureGroup = internalReverseScan(startKey, realEndKey, readOnlySafe,
+            returnValue, this.failoverRetries, null);
+        return FutureHelper.joinList(futureGroup);
+    }
+
+    @Override
+    public CompletableFuture> reverseScan(final String startKey, final String endKey,
+                                                        final boolean readOnlySafe, final boolean returnValue) {
+        return reverseScan(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), readOnlySafe, returnValue);
+    }
+
+    @Override
+    public List bReverseScan(final byte[] startKey, final byte[] endKey) {
+        return FutureHelper.get(reverseScan(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bReverseScan(final String startKey, final String endKey) {
+        return FutureHelper.get(reverseScan(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bReverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe) {
+        return FutureHelper.get(reverseScan(startKey, endKey, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bReverseScan(final String startKey, final String endKey, final boolean readOnlySafe) {
+        return FutureHelper.get(reverseScan(startKey, endKey, readOnlySafe), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bReverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                                      final boolean returnValue) {
+        return FutureHelper.get(reverseScan(startKey, endKey, readOnlySafe, returnValue), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public List bReverseScan(final String startKey, final String endKey, final boolean readOnlySafe,
+                                      final boolean returnValue) {
+        return FutureHelper.get(reverseScan(startKey, endKey, readOnlySafe, returnValue), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup> internalReverseScan(final byte[] startKey, final byte[] endKey,
+                                                           final boolean readOnlySafe, final boolean returnValue,
+                                                           final int retriesLeft, final Throwable lastCause) {
+        Requires.requireNonNull(endKey, "endKey");
+        final List regionList = this.pdClient.findRegionsByKeyRange(endKey, startKey, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        Collections.reverse(regionList);
+        final List>> futures = Lists.newArrayListWithCapacity(regionList.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Region region : regionList) {
+            final byte[] regionEndKey = region.getEndKey();
+            final byte[] regionStartKey = region.getStartKey();
+            final byte[] subStartKey = regionEndKey == null ? startKey : (startKey == null ? regionEndKey : BytesUtil.min(regionEndKey, startKey));
+            final byte[] subEndKey = regionStartKey == null ? endKey : BytesUtil.max(regionStartKey, endKey);
+            final ListRetryCallable retryCallable = retryCause -> internalReverseScan(subStartKey, subEndKey,
+                    readOnlySafe, returnValue, retriesLeft - 1, retryCause);
+            final ListFailoverFuture future = new ListFailoverFuture<>(retriesLeft, retryCallable);
+            internalRegionScan(region, subStartKey, subEndKey, true, readOnlySafe, returnValue, future, retriesLeft,
+                    lastError, this.onlyLeaderRead );
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    public List singleRegionScan(final byte[] startKey, final byte[] endKey, final int limit,
+                                          final boolean readOnlySafe, final boolean returnValue) {
+        checkState();
+        final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+        if (endKey != null) {
+            Requires.requireTrue(BytesUtil.compare(realStartKey, endKey) < 0, "startKey must < endKey");
+        }
+        Requires.requireTrue(limit > 0, "limit must > 0");
+        final CompletableFuture> future = new CompletableFuture<>();
+        internalSingleRegionScan(realStartKey, endKey, limit, readOnlySafe, returnValue, future, this.failoverRetries,
+            null, this.onlyLeaderRead);
+        return FutureHelper.get(future, this.futureTimeoutMillis);
+    }
+
+    private void internalSingleRegionScan(final byte[] startKey, final byte[] endKey, final int limit,
+                                          final boolean readOnlySafe, final boolean returnValue,
+                                          final CompletableFuture> future, final int retriesLeft,
+                                          final Errors lastCause, final boolean requireLeader) {
+        Requires.requireNonNull(startKey, "startKey");
+        final Region region = this.pdClient.findRegionByKey(startKey, ErrorsHelper.isInvalidEpoch(lastCause));
+        final byte[] regionEndKey = region.getEndKey();
+        final byte[] realEndKey = regionEndKey == null ? endKey :
+                (endKey == null ? regionEndKey : BytesUtil.min(regionEndKey, endKey));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), requireLeader);
+        // require leader on retry
+        final RetryRunner retryRunner = retryCause -> internalSingleRegionScan(startKey, endKey, limit, readOnlySafe,
+                returnValue, future, retriesLeft - 1, retryCause, true);
+        final FailoverClosure> closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).scan(startKey, realEndKey, limit, readOnlySafe, returnValue, closure);
+            }
+        } else {
+            final ScanRequest request = new ScanRequest();
+            request.setStartKey(startKey);
+            request.setEndKey(realEndKey);
+            request.setLimit(limit);
+            request.setReadOnlySafe(readOnlySafe);
+            request.setReturnValue(returnValue);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause, requireLeader);
+        }
+    }
+
+    @Override
+    public RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize) {
+        return iterator(startKey, endKey, bufSize, true);
+    }
+
+    @Override
+    public RheaIterator iterator(final String startKey, final String endKey, final int bufSize) {
+        return iterator(startKey, endKey, bufSize, true);
+    }
+
+    @Override
+    public RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize,
+                                          final boolean readOnlySafe) {
+        return iterator(startKey, endKey, bufSize, readOnlySafe, true);
+    }
+
+    @Override
+    public RheaIterator iterator(final String startKey, final String endKey, final int bufSize,
+                                          final boolean readOnlySafe) {
+        return iterator(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), bufSize, readOnlySafe);
+    }
+
+    @Override
+    public RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize,
+                                          final boolean readOnlySafe, final boolean returnValue) {
+        return new DefaultRheaIterator(this, startKey, endKey, bufSize, readOnlySafe, returnValue);
+    }
+
+    @Override
+    public RheaIterator iterator(final String startKey, final String endKey, final int bufSize,
+                                          final boolean readOnlySafe, final boolean returnValue) {
+        return iterator(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey), bufSize, readOnlySafe, returnValue);
+    }
+
+    @Override
+    public CompletableFuture getSequence(final byte[] seqKey, final int step) {
+        checkState();
+        Requires.requireNonNull(seqKey, "seqKey");
+        Requires.requireTrue(step >= 0, "step must >= 0");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalGetSequence(seqKey, step, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture getSequence(final String seqKey, final int step) {
+        return getSequence(BytesUtil.writeUtf8(seqKey), step);
+    }
+
+    @Override
+    public Sequence bGetSequence(final byte[] seqKey, final int step) {
+        return FutureHelper.get(getSequence(seqKey, step), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Sequence bGetSequence(final String seqKey, final int step) {
+        return FutureHelper.get(getSequence(seqKey, step), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public CompletableFuture getLatestSequence(final byte[] seqKey) {
+        final CompletableFuture cf = new CompletableFuture<>();
+        getSequence(seqKey, 0).whenComplete((sequence, throwable) -> {
+            if (throwable == null) {
+                cf.complete(sequence.getStartValue());
+            } else {
+                cf.completeExceptionally(throwable);
+            }
+        });
+        return cf;
+    }
+
+    @Override
+    public CompletableFuture getLatestSequence(final String seqKey) {
+        return getLatestSequence(BytesUtil.writeUtf8(seqKey));
+    }
+
+    @Override
+    public Long bGetLatestSequence(final byte[] seqKey) {
+        return FutureHelper.get(getLatestSequence(seqKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Long bGetLatestSequence(final String seqKey) {
+        return FutureHelper.get(getLatestSequence(seqKey), this.futureTimeoutMillis);
+    }
+
+    private void internalGetSequence(final byte[] seqKey, final int step, final CompletableFuture future,
+                                     final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(seqKey, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalGetSequence(seqKey, step, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).getSequence(seqKey, step, closure);
+            }
+        } else {
+            final GetSequenceRequest request = new GetSequenceRequest();
+            request.setSeqKey(seqKey);
+            request.setStep(step);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture resetSequence(final byte[] seqKey) {
+        checkState();
+        Requires.requireNonNull(seqKey, "seqKey");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalResetSequence(seqKey, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture resetSequence(final String seqKey) {
+        return resetSequence(BytesUtil.writeUtf8(seqKey));
+    }
+
+    @Override
+    public Boolean bResetSequence(final byte[] seqKey) {
+        return FutureHelper.get(resetSequence(seqKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bResetSequence(final String seqKey) {
+        return FutureHelper.get(resetSequence(seqKey), this.futureTimeoutMillis);
+    }
+
+    private void internalResetSequence(final byte[] seqKey, final CompletableFuture future,
+                                       final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(seqKey, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalResetSequence(seqKey, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).resetSequence(seqKey, closure);
+            }
+        } else {
+            final ResetSequenceRequest request = new ResetSequenceRequest();
+            request.setSeqKey(seqKey);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture put(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return put(key, value, new CompletableFuture<>(), true);
+    }
+
+    @Override
+    public CompletableFuture put(final String key, final byte[] value) {
+        return put(BytesUtil.writeUtf8(key), value);
+    }
+
+    @Override
+    public Boolean bPut(final byte[] key, final byte[] value) {
+        return FutureHelper.get(put(key, value), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bPut(final String key, final byte[] value) {
+        return FutureHelper.get(put(key, value), this.futureTimeoutMillis);
+    }
+
+    private CompletableFuture put(final byte[] key, final byte[] value,
+                                           final CompletableFuture future, final boolean tryBatching) {
+        checkState();
+        if (tryBatching) {
+            final PutBatching putBatching = this.putBatching;
+            if (putBatching != null && putBatching.apply(new KVEntry(key, value), future)) {
+                return future;
+            }
+        }
+        internalPut(key, value, future, this.failoverRetries, null);
+        return future;
+    }
+
+    private void internalPut(final byte[] key, final byte[] value, final CompletableFuture future,
+                             final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalPut(key, value, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).put(key, value, closure);
+            }
+        } else {
+            final PutRequest request = new PutRequest();
+            request.setKey(key);
+            request.setValue(value);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture getAndPut(final byte[] key, final byte[] value) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalGetAndPut(key, value, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture getAndPut(final String key, final byte[] value) {
+        return getAndPut(BytesUtil.writeUtf8(key), value);
+    }
+
+    @Override
+    public byte[] bGetAndPut(final byte[] key, final byte[] value) {
+        return FutureHelper.get(getAndPut(key, value), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public byte[] bGetAndPut(final String key, final byte[] value) {
+        return FutureHelper.get(getAndPut(key, value), this.futureTimeoutMillis);
+    }
+
+    private void internalGetAndPut(final byte[] key, final byte[] value, final CompletableFuture future,
+                                   final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalGetAndPut(key, value, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).getAndPut(key, value, closure);
+            }
+        } else {
+            final GetAndPutRequest request = new GetAndPutRequest();
+            request.setKey(key);
+            request.setValue(value);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture compareAndPut(final byte[] key, final byte[] expect, final byte[] update) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(expect, "expect");
+        Requires.requireNonNull(update, "update");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalCompareAndPut(key, expect, update, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture compareAndPut(final String key, final byte[] expect, final byte[] update) {
+        return compareAndPut(BytesUtil.writeUtf8(key), expect, update);
+    }
+
+    @Override
+    public Boolean bCompareAndPut(final byte[] key, final byte[] expect, final byte[] update) {
+        return FutureHelper.get(compareAndPut(key, expect, update), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bCompareAndPut(final String key, final byte[] expect, final byte[] update) {
+        return FutureHelper.get(compareAndPut(key, expect, update), this.futureTimeoutMillis);
+    }
+
+    private void internalCompareAndPut(final byte[] key, final byte[] expect, final byte[] update,
+                                       final CompletableFuture future, final int retriesLeft,
+                                       final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalCompareAndPut(key, expect, update, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).compareAndPut(key, expect, update, closure);
+            }
+        } else {
+            final CompareAndPutRequest request = new CompareAndPutRequest();
+            request.setKey(key);
+            request.setExpect(expect);
+            request.setUpdate(update);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture merge(final String key, final String value) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalMerge(BytesUtil.writeUtf8(key), BytesUtil.writeUtf8(value), future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public Boolean bMerge(final String key, final String value) {
+        return FutureHelper.get(merge(key, value), this.futureTimeoutMillis);
+    }
+
+    private void internalMerge(final byte[] key, final byte[] value, final CompletableFuture future,
+                               final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalMerge(key, value, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).merge(key, value, closure);
+            }
+        } else {
+            final MergeRequest request = new MergeRequest();
+            request.setKey(key);
+            request.setValue(value);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    // Note: the current implementation, if the 'keys' are distributed across
+    // multiple regions, can not provide transaction guarantee.
+    @Override
+    public CompletableFuture put(final List entries) {
+        checkState();
+        Requires.requireNonNull(entries, "entries");
+        Requires.requireTrue(!entries.isEmpty(), "entries empty");
+        final FutureGroup futureGroup = internalPut(entries, this.failoverRetries, null);
+        return FutureHelper.joinBooleans(futureGroup);
+    }
+
+    @Override
+    public Boolean bPut(final List entries) {
+        return FutureHelper.get(put(entries), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup internalPut(final List entries, final int retriesLeft,
+                                             final Throwable lastCause) {
+        final Map> regionMap = this.pdClient
+                .findRegionsByKvEntries(entries, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List> futures = Lists.newArrayListWithCapacity(regionMap.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Map.Entry> entry : regionMap.entrySet()) {
+            final Region region = entry.getKey();
+            final List subEntries = entry.getValue();
+            final RetryCallable retryCallable = retryCause -> internalPut(subEntries, retriesLeft - 1,
+                    retryCause);
+            final BoolFailoverFuture future = new BoolFailoverFuture(retriesLeft, retryCallable);
+            internalRegionPut(region, subEntries, future, retriesLeft, lastError);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    private void internalRegionPut(final Region region, final List subEntries,
+                                   final CompletableFuture future, final int retriesLeft,
+                                   final Errors lastCause) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalRegionPut(region, subEntries, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, false, retriesLeft,
+                retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                final RawKVStore rawKVStore = getRawKVStore(regionEngine);
+                if (this.kvDispatcher == null) {
+                    rawKVStore.put(subEntries, closure);
+                } else {
+                    this.kvDispatcher.execute(() -> rawKVStore.put(subEntries, closure));
+                }
+            }
+        } else {
+            final BatchPutRequest request = new BatchPutRequest();
+            request.setKvEntries(subEntries);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    // Note: the current implementation, if the 'keys' are distributed across
+    // multiple regions, can not provide transaction guarantee.
+    @Override
+    public CompletableFuture compareAndPutAll(final List entries) {
+        checkState();
+        Requires.requireNonNull(entries, "entries");
+        Requires.requireTrue(!entries.isEmpty(), "entries empty");
+        final FutureGroup futureGroup = internalCompareAndPutAll(entries, this.failoverRetries, null);
+        return FutureHelper.joinBooleans(futureGroup);
+    }
+
+    @Override
+    public Boolean bCompareAndPutAll(final List entries) {
+        return FutureHelper.get(compareAndPutAll(entries), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup internalCompareAndPutAll(final List entries, final int retriesLeft,
+                                                          final Throwable lastCause) {
+        final Map> regionMap = this.pdClient
+                .findRegionsByCASEntries(entries, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List> futures = Lists.newArrayListWithCapacity(regionMap.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Map.Entry> entry : regionMap.entrySet()) {
+            final Region region = entry.getKey();
+            final List subEntries = entry.getValue();
+            final RetryCallable retryCallable = retryCause -> internalCompareAndPutAll(subEntries,
+                    retriesLeft - 1, retryCause);
+            final BoolFailoverFuture future = new BoolFailoverFuture(retriesLeft, retryCallable);
+            internalRegionCompareAndPutAll(region, subEntries, future, retriesLeft, lastError);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    private void internalRegionCompareAndPutAll(final Region region, final List subEntries,
+                                                final CompletableFuture future, final int retriesLeft,
+                                                final Errors lastCause) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalRegionCompareAndPutAll(region, subEntries, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, false, retriesLeft,
+                retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                final RawKVStore rawKVStore = getRawKVStore(regionEngine);
+                if (this.kvDispatcher == null) {
+                    rawKVStore.compareAndPutAll(subEntries, closure);
+                } else {
+                    this.kvDispatcher.execute(() -> rawKVStore.compareAndPutAll(subEntries, closure));
+                }
+            }
+        } else {
+            final CASAllRequest request = new CASAllRequest();
+            request.setCasEntries(subEntries);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture putIfAbsent(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalPutIfAbsent(key, value, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture putIfAbsent(final String key, final byte[] value) {
+        return putIfAbsent(BytesUtil.writeUtf8(key), value);
+    }
+
+    @Override
+    public byte[] bPutIfAbsent(final byte[] key, final byte[] value) {
+        return FutureHelper.get(putIfAbsent(key, value), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public byte[] bPutIfAbsent(final String key, final byte[] value) {
+        return FutureHelper.get(putIfAbsent(key, value), this.futureTimeoutMillis);
+    }
+
+    private void internalPutIfAbsent(final byte[] key, final byte[] value, final CompletableFuture future,
+                                     final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalPutIfAbsent(key, value, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).putIfAbsent(key, value, closure);
+            }
+        } else {
+            final PutIfAbsentRequest request = new PutIfAbsentRequest();
+            request.setKey(key);
+            request.setValue(value);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture delete(final byte[] key) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalDelete(key, future, this.failoverRetries, null);
+        return future;
+    }
+
+    @Override
+    public CompletableFuture delete(final String key) {
+        return delete(BytesUtil.writeUtf8(key));
+    }
+
+    @Override
+    public Boolean bDelete(final byte[] key) {
+        return FutureHelper.get(delete(key), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bDelete(final String key) {
+        return FutureHelper.get(delete(key), this.futureTimeoutMillis);
+    }
+
+    private void internalDelete(final byte[] key, final CompletableFuture future, final int retriesLeft,
+                                final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalDelete(key, future, retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).delete(key, closure);
+            }
+        } else {
+            final DeleteRequest request = new DeleteRequest();
+            request.setKey(key);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public CompletableFuture deleteRange(final byte[] startKey, final byte[] endKey) {
+        checkState();
+        Requires.requireNonNull(startKey, "startKey");
+        Requires.requireNonNull(endKey, "endKey");
+        Requires.requireTrue(BytesUtil.compare(startKey, endKey) < 0, "startKey must < endKey");
+        final FutureGroup futureGroup = internalDeleteRange(startKey, endKey, this.failoverRetries, null);
+        return FutureHelper.joinBooleans(futureGroup);
+    }
+
+    private FutureGroup internalDeleteRange(final byte[] startKey, final byte[] endKey, final int retriesLeft,
+                                                     final Throwable lastCause) {
+        final List regionList = this.pdClient
+                .findRegionsByKeyRange(startKey, endKey, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List> futures = Lists.newArrayListWithCapacity(regionList.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Region region : regionList) {
+            final byte[] regionStartKey = region.getStartKey();
+            final byte[] regionEndKey = region.getEndKey();
+            final byte[] subStartKey = regionStartKey == null ? startKey : BytesUtil.max(regionStartKey, startKey);
+            final byte[] subEndKey = regionEndKey == null ? endKey : BytesUtil.min(regionEndKey, endKey);
+            final RetryCallable retryCallable = retryCause -> internalDeleteRange(subStartKey, subEndKey,
+                    retriesLeft - 1, retryCause);
+            final BoolFailoverFuture future = new BoolFailoverFuture(retriesLeft, retryCallable);
+            internalRegionDeleteRange(region, subStartKey, subEndKey, future, retriesLeft, lastError);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    @Override
+    public CompletableFuture deleteRange(final String startKey, final String endKey) {
+        return deleteRange(BytesUtil.writeUtf8(startKey), BytesUtil.writeUtf8(endKey));
+    }
+
+    @Override
+    public Boolean bDeleteRange(final byte[] startKey, final byte[] endKey) {
+        return FutureHelper.get(deleteRange(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public Boolean bDeleteRange(final String startKey, final String endKey) {
+        return FutureHelper.get(deleteRange(startKey, endKey), this.futureTimeoutMillis);
+    }
+
+    @Override
+    public CompletableFuture delete(final List keys) {
+        checkState();
+        Requires.requireNonNull(keys, "keys");
+        Requires.requireTrue(!keys.isEmpty(), "keys empty");
+        final FutureGroup futureGroup = internalDelete(keys, this.failoverRetries, null);
+        return FutureHelper.joinBooleans(futureGroup);
+    }
+
+    @Override
+    public Boolean bDelete(final List keys) {
+        return FutureHelper.get(delete(keys), this.futureTimeoutMillis);
+    }
+
+    private FutureGroup internalDelete(final List keys, final int retriesLeft,
+                                                final Throwable lastCause) {
+        final Map> regionMap = this.pdClient
+                .findRegionsByKeys(keys, ApiExceptionHelper.isInvalidEpoch(lastCause));
+        final List> futures = Lists.newArrayListWithCapacity(regionMap.size());
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        for (final Map.Entry> entry : regionMap.entrySet()) {
+            final Region region = entry.getKey();
+            final List subKeys = entry.getValue();
+            final RetryCallable retryCallable = retryCause -> internalDelete(subKeys, retriesLeft - 1,
+                    retryCause);
+            final BoolFailoverFuture future = new BoolFailoverFuture(retriesLeft, retryCallable);
+            internalRegionDelete(region, subKeys, future, retriesLeft, lastError);
+            futures.add(future);
+        }
+        return new FutureGroup<>(futures);
+    }
+
+    private void internalRegionDelete(final Region region, final List subKeys,
+                                      final CompletableFuture future, final int retriesLeft,
+                                      final Errors lastCause) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalRegionDelete(region, subKeys, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, false, retriesLeft,
+                retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                final RawKVStore rawKVStore = getRawKVStore(regionEngine);
+                if (this.kvDispatcher == null) {
+                    rawKVStore.delete(subKeys, closure);
+                } else {
+                    this.kvDispatcher.execute(() -> rawKVStore.delete(subKeys, closure));
+                }
+            }
+        } else {
+            final BatchDeleteRequest request = new BatchDeleteRequest();
+            request.setKeys(subKeys);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    private void internalRegionDeleteRange(final Region region, final byte[] subStartKey, final byte[] subEndKey,
+                                           final CompletableFuture future, final int retriesLeft,
+                                           final Errors lastCause) {
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalRegionDeleteRange(region, subStartKey, subEndKey, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure =
+                new FailoverClosureImpl<>(future, false, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).deleteRange(subStartKey, subEndKey, closure);
+            }
+        } else {
+            final DeleteRangeRequest request = new DeleteRangeRequest();
+            request.setStartKey(subStartKey);
+            request.setEndKey(subEndKey);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    // internal api
+    public CompletableFuture execute(final long regionId, final NodeExecutor executor) {
+        checkState();
+        Requires.requireNonNull(executor, "executor");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalExecute(regionId, executor, future, this.failoverRetries, null);
+        return future;
+    }
+
+    // internal api
+    public Boolean bExecute(final long regionId, final NodeExecutor executor) {
+        return FutureHelper.get(execute(regionId, executor), this.futureTimeoutMillis);
+    }
+
+    private void internalExecute(final long regionId, final NodeExecutor executor,
+                                 final CompletableFuture future, final int retriesLeft, final Errors lastCause) {
+        final Region region = this.pdClient.getRegionById(regionId);
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalExecute(regionId, executor, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).execute(executor, true, closure);
+            }
+        } else {
+            final NodeExecuteRequest request = new NodeExecuteRequest();
+            request.setNodeExecutor(executor);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public DistributedLock getDistributedLock(final byte[] target, final long lease, final TimeUnit unit) {
+        return getDistributedLock(target, lease, unit, null);
+    }
+
+    @Override
+    public DistributedLock getDistributedLock(final String target, final long lease, final TimeUnit unit) {
+        return getDistributedLock(target, lease, unit, null);
+    }
+
+    @Override
+    public DistributedLock getDistributedLock(final byte[] target, final long lease, final TimeUnit unit,
+                                                      final ScheduledExecutorService watchdog) {
+        return new DefaultDistributedLock(target, lease, unit, watchdog, this);
+    }
+
+    @Override
+    public DistributedLock getDistributedLock(final String target, final long lease, final TimeUnit unit,
+                                                      final ScheduledExecutorService watchdog) {
+        return getDistributedLock(BytesUtil.writeUtf8(target), lease, unit, watchdog);
+    }
+
+    public CompletableFuture tryLockWith(final byte[] key, final boolean keepLease,
+                                                                final DistributedLock.Acquirer acquirer) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalTryLockWith(key, keepLease, acquirer, future, this.failoverRetries, null);
+        return future;
+    }
+
+    private void internalTryLockWith(final byte[] key, final boolean keepLease, final DistributedLock.Acquirer acquirer,
+                                     final CompletableFuture future, final int retriesLeft,
+                                     final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalTryLockWith(key, keepLease, acquirer, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft,
+                retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).tryLockWith(key, region.getStartKey(), keepLease, acquirer, closure);
+            }
+        } else {
+            final KeyLockRequest request = new KeyLockRequest();
+            request.setKey(key);
+            request.setKeepLease(keepLease);
+            request.setAcquirer(acquirer);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    public CompletableFuture releaseLockWith(final byte[] key,
+                                                                    final DistributedLock.Acquirer acquirer) {
+        checkState();
+        Requires.requireNonNull(key, "key");
+        final CompletableFuture future = new CompletableFuture<>();
+        internalReleaseLockWith(key, acquirer, future, this.failoverRetries, null);
+        return future;
+    }
+
+    private void internalReleaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer,
+                                         final CompletableFuture future, final int retriesLeft,
+                                         final Errors lastCause) {
+        final Region region = this.pdClient.findRegionByKey(key, ErrorsHelper.isInvalidEpoch(lastCause));
+        final RegionEngine regionEngine = getRegionEngine(region.getId(), true);
+        final RetryRunner retryRunner = retryCause -> internalReleaseLockWith(key, acquirer, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft,
+                retryRunner);
+        if (regionEngine != null) {
+            if (ensureOnValidEpoch(region, regionEngine, closure)) {
+                getRawKVStore(regionEngine).releaseLockWith(key, acquirer, closure);
+            }
+        } else {
+            final KeyUnlockRequest request = new KeyUnlockRequest();
+            request.setKey(key);
+            request.setAcquirer(acquirer);
+            request.setRegionId(region.getId());
+            request.setRegionEpoch(region.getRegionEpoch());
+            this.rheaKVRpcService.callAsyncWithRpc(request, closure, lastCause);
+        }
+    }
+
+    @Override
+    public PlacementDriverClient getPlacementDriverClient() {
+        return pdClient;
+    }
+
+    @Override
+    public void addLeaderStateListener(final long regionId, final LeaderStateListener listener) {
+        addStateListener(regionId, listener);
+    }
+
+    @Override
+    public void addFollowerStateListener(final long regionId, final FollowerStateListener listener) {
+        addStateListener(regionId, listener);
+    }
+
+    @Override
+    public void addStateListener(final long regionId, final StateListener listener) {
+        this.stateListenerContainer.addStateListener(regionId, listener);
+    }
+
+    public long getClusterId() {
+        return this.opts.getClusterId();
+    }
+
+    public StoreEngine getStoreEngine() {
+        return storeEngine;
+    }
+
+    public boolean isOnlyLeaderRead() {
+        return onlyLeaderRead;
+    }
+
+    public boolean isLeader(final long regionId) {
+        checkState();
+        final RegionEngine regionEngine = getRegionEngine(regionId);
+        return regionEngine != null && regionEngine.isLeader();
+    }
+
+    private void checkState() {
+        // Not a strict state check, more is to use a read volatile operation to make
+        // a happen-before, because the init() method finally wrote 'this.started'
+        if (!this.started) {
+            throw new RheaRuntimeException("rhea kv is not started or shutdown");
+        }
+    }
+
+    private RegionEngine getRegionEngine(final long regionId) {
+        if (this.storeEngine == null) {
+            return null;
+        }
+        return this.storeEngine.getRegionEngine(regionId);
+    }
+
+    private RegionEngine getRegionEngine(final long regionId, final boolean requireLeader) {
+        final RegionEngine engine = getRegionEngine(regionId);
+        if (engine == null) {
+            return null;
+        }
+        if (requireLeader && !engine.isLeader()) {
+            return null;
+        }
+        return engine;
+    }
+
+    private Endpoint getLeaderByRegionEngine(final long regionId) {
+        final RegionEngine regionEngine = getRegionEngine(regionId);
+        if (regionEngine != null) {
+            final PeerId leader = regionEngine.getLeaderId();
+            if (leader != null) {
+                final String raftGroupId = JRaftHelper.getJRaftGroupId(this.pdClient.getClusterName(), regionId);
+                RouteTable.getInstance().updateLeader(raftGroupId, leader);
+                return leader.getEndpoint();
+            }
+        }
+        return null;
+    }
+
+    private RawKVStore getRawKVStore(final RegionEngine engine) {
+        return engine.getMetricsRawKVStore();
+    }
+
+    private static boolean ensureOnValidEpoch(final Region region, final RegionEngine engine,
+                                              final KVStoreClosure closure) {
+        if (isValidEpoch(region, engine)) {
+            return true;
+        }
+        // will retry on this error and status
+        closure.setError(Errors.INVALID_REGION_EPOCH);
+        closure.run(new Status(-1, "Invalid region epoch: %s", region));
+        return false;
+    }
+
+    private static boolean isValidEpoch(final Region region, final RegionEngine engine) {
+        return region.getRegionEpoch().equals(engine.getRegion().getRegionEpoch());
+    }
+
+    private class GetBatching extends Batching {
+
+        public GetBatching(EventFactory factory, String name, EventHandler handler) {
+            super(factory, batchingOpts.getBufSize(), name, handler);
+        }
+
+        @Override
+        public boolean apply(final byte[] message, final CompletableFuture future) {
+            return this.ringBuffer.tryPublishEvent((event, sequence) -> {
+                event.reset();
+                event.key = message;
+                event.future = future;
+            });
+        }
+    }
+
+    private class PutBatching extends Batching {
+
+        public PutBatching(EventFactory factory, String name, PutBatchingHandler handler) {
+            super(factory, batchingOpts.getBufSize(), name, handler);
+        }
+
+        @Override
+        public boolean apply(final KVEntry message, final CompletableFuture future) {
+            return this.ringBuffer.tryPublishEvent((event, sequence) -> {
+                event.reset();
+                event.kvEntry = message;
+                event.future = future;
+            });
+        }
+    }
+
+    private class GetBatchingHandler extends AbstractBatchingHandler {
+
+        private final boolean readOnlySafe;
+
+        private GetBatchingHandler(String metricsName, boolean readOnlySafe) {
+            super(metricsName);
+            this.readOnlySafe = readOnlySafe;
+        }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public void onEvent(final KeyEvent event, final long sequence, final boolean endOfBatch) throws Exception {
+            this.events.add(event);
+            this.cachedBytes += event.key.length;
+            final int size = this.events.size();
+            if (!endOfBatch && size < batchingOpts.getBatchSize() && this.cachedBytes < batchingOpts.getMaxReadBytes()) {
+                return;
+            }
+
+            if (size == 1) {
+                try {
+                    get(event.key, this.readOnlySafe, event.future, false);
+                } catch (final Throwable t) {
+                    exceptionally(t, event.future);
+                }
+                reset();
+            } else {
+                final List keys = Lists.newArrayListWithCapacity(size);
+                final CompletableFuture[] futures = new CompletableFuture[size];
+                for (int i = 0; i < size; i++) {
+                    final KeyEvent e = this.events.get(i);
+                    keys.add(e.key);
+                    futures[i] = e.future;
+                }
+                reset();
+                try {
+                    multiGet(keys, this.readOnlySafe).whenComplete((result, throwable) -> {
+                        if (throwable == null) {
+                            for (int i = 0; i < futures.length; i++) {
+                                final ByteArray realKey = ByteArray.wrap(keys.get(i));
+                                futures[i].complete(result.get(realKey));
+                            }
+                            return;
+                        }
+                        exceptionally(throwable, futures);
+                    });
+                } catch (final Throwable t) {
+                    exceptionally(t, futures);
+                }
+            }
+        }
+    }
+
+    private class PutBatchingHandler extends AbstractBatchingHandler {
+
+        public PutBatchingHandler(String metricsName) {
+            super(metricsName);
+        }
+
+        @SuppressWarnings("unchecked")
+        @Override
+        public void onEvent(final KVEvent event, final long sequence, final boolean endOfBatch) throws Exception {
+            this.events.add(event);
+            this.cachedBytes += event.kvEntry.length();
+            final int size = this.events.size();
+            if (!endOfBatch && size < batchingOpts.getBatchSize() && this.cachedBytes < batchingOpts.getMaxWriteBytes()) {
+                return;
+            }
+
+            if (size == 1) {
+                final KVEntry kv = event.kvEntry;
+                try {
+                    put(kv.getKey(), kv.getValue(), event.future, false);
+                } catch (final Throwable t) {
+                    exceptionally(t, event.future);
+                }
+                reset();
+            } else {
+                final List entries = Lists.newArrayListWithCapacity(size);
+                final CompletableFuture[] futures = new CompletableFuture[size];
+                for (int i = 0; i < size; i++) {
+                    final KVEvent e = this.events.get(i);
+                    entries.add(e.kvEntry);
+                    futures[i] = e.future;
+                }
+                reset();
+                try {
+                    put(entries).whenComplete((result, throwable) -> {
+                        if (throwable == null) {
+                            for (int i = 0; i < futures.length; i++) {
+                                futures[i].complete(result);
+                            }
+                            return;
+                        }
+                        exceptionally(throwable, futures);
+                    });
+                } catch (final Throwable t) {
+                    exceptionally(t, futures);
+                }
+            }
+        }
+    }
+
+    private abstract class AbstractBatchingHandler implements EventHandler {
+
+        protected final Histogram histogramWithKeys;
+        protected final Histogram histogramWithBytes;
+
+        protected final List   events      = Lists.newArrayListWithCapacity(batchingOpts.getBatchSize());
+        protected int             cachedBytes = 0;
+
+        public AbstractBatchingHandler(String metricsName) {
+            this.histogramWithKeys = KVMetrics.histogram(KVMetricNames.SEND_BATCHING, metricsName + "_keys");
+            this.histogramWithBytes = KVMetrics.histogram(KVMetricNames.SEND_BATCHING, metricsName + "_bytes");
+        }
+
+        public void exceptionally(final Throwable t, final CompletableFuture... futures) {
+            for (int i = 0; i < futures.length; i++) {
+                futures[i].completeExceptionally(t);
+            }
+        }
+
+        public void reset() {
+            this.histogramWithKeys.update(this.events.size());
+            this.histogramWithBytes.update(this.cachedBytes);
+
+            for (final T event : events) {
+                event.reset();
+            }
+            this.events.clear();
+            this.cachedBytes = 0;
+        }
+
+    }
+
+    private interface Event {
+        void reset();
+    }
+
+    private static class KeyEvent implements Event {
+
+        private byte[]                    key;
+        private CompletableFuture future;
+
+        @Override
+        public void reset() {
+            this.key = null;
+            this.future = null;
+        }
+    }
+
+    private static class KVEvent implements Event {
+
+        private KVEntry                    kvEntry;
+        private CompletableFuture future;
+
+        @Override
+        public void reset() {
+            this.kvEntry = null;
+            this.future = null;
+        }
+    }
+
+    private static abstract class Batching {
+
+        protected final String        name;
+        protected final Disruptor  disruptor;
+        protected final RingBuffer ringBuffer;
+
+        @SuppressWarnings("unchecked")
+        public Batching(EventFactory factory, int bufSize, String name, EventHandler handler) {
+            this.name = name;
+            this.disruptor = new Disruptor<>(factory, bufSize, new NamedThreadFactory(name, true));
+            this.disruptor.handleEventsWith(handler);
+            this.disruptor.setDefaultExceptionHandler(new LogExceptionHandler(name));
+            this.ringBuffer = this.disruptor.start();
+        }
+
+        public abstract boolean apply(final E message, final CompletableFuture future);
+
+        public void shutdown() {
+            try {
+                this.disruptor.shutdown(3L, TimeUnit.SECONDS);
+            } catch (final Exception e) {
+                LOG.error("Fail to shutdown {}, {}.", toString(), StackTraceUtil.stackTrace(e));
+            }
+        }
+
+        @Override
+        public String toString() {
+            return "Batching{" + "name='" + name + '\'' + ", disruptor=" + disruptor + '}';
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureGroup.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureGroup.java
new file mode 100644
index 0000000..555b55d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureGroup.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class FutureGroup extends CompletableFuture {
+
+    private final List> futures;
+
+    private volatile CompletableFuture[]  array;
+
+    public FutureGroup(List> futures) {
+        this.futures = Requires.requireNonNull(futures, "futures");
+    }
+
+    public List> futures() {
+        return futures;
+    }
+
+    @SuppressWarnings("unchecked")
+    public CompletableFuture[] toArray() {
+        if (this.array == null) {
+            synchronized (this) {
+                if (this.array == null) {
+                    final CompletableFuture[] temp = new CompletableFuture[this.futures.size()];
+                    this.futures.toArray(temp);
+                    this.array = temp;
+                }
+            }
+        }
+        return this.array;
+    }
+
+    public int size() {
+        return this.futures.size();
+    }
+
+    @Override
+    public boolean cancel(final boolean mayInterruptIfRunning) {
+        boolean result = true;
+        for (final CompletableFuture f : this.futures) {
+            result = result && f.cancel(mayInterruptIfRunning);
+        }
+        return result;
+    }
+
+    @Override
+    public boolean isCancelled() {
+        for (final CompletableFuture f : this.futures) {
+            if (!f.isCancelled()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean isDone() {
+        for (final CompletableFuture f : this.futures) {
+            if (!f.isDone()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public V get() throws InterruptedException, ExecutionException {
+        throw new UnsupportedOperationException("get");
+    }
+
+    @Override
+    public V get(final long timeout, final TimeUnit unit) throws InterruptedException, ExecutionException,
+                                                         TimeoutException {
+        throw new UnsupportedOperationException("get");
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureHelper.java
new file mode 100644
index 0000000..6904b03
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/FutureHelper.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.alipay.sofa.jraft.rhea.errors.NeverGetHereException;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class FutureHelper {
+
+    public static final long DEFAULT_TIMEOUT_MILLIS = SystemPropertyUtil.getLong("rhea.default_future_timeout", 10000);
+
+    public static  V get(final CompletableFuture future) {
+        return get(future, DEFAULT_TIMEOUT_MILLIS);
+    }
+
+    public static  V get(final CompletableFuture future, final long timeoutMillis) {
+        try {
+            return future.get(timeoutMillis, TimeUnit.MILLISECONDS);
+        } catch (final InterruptedException | ExecutionException | TimeoutException e) {
+            ThrowUtil.throwException(e);
+        }
+        throw NeverGetHereException.INSTANCE;
+    }
+
+    public static CompletableFuture joinBooleans(final FutureGroup futureGroup) {
+        return joinBooleans(futureGroup, new CompletableFuture<>());
+    }
+
+    public static CompletableFuture joinBooleans(final FutureGroup futureGroup,
+                                                          final CompletableFuture future) {
+        CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+            if (throwable == null) {
+                for (final CompletableFuture partOf : futureGroup.futures()) {
+                    if (!partOf.join()) {
+                        future.complete(false);
+                        return;
+                    }
+                }
+                future.complete(true);
+            } else {
+                future.completeExceptionally(throwable);
+            }
+        });
+        return future;
+    }
+
+    public static  CompletableFuture> joinList(final FutureGroup> futureGroup) {
+        return joinList(futureGroup, 0);
+    }
+
+    public static  CompletableFuture> joinList(final FutureGroup> futureGroup, final int size) {
+        return joinList(futureGroup, size, new CompletableFuture<>());
+    }
+
+    public static  CompletableFuture> joinList(final FutureGroup> futureGroup, final int size,
+                                                          final CompletableFuture> future) {
+        CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+            if (throwable == null) {
+                final List allResult = size > 0 ? Lists.newArrayListWithCapacity(size) : Lists.newArrayList();
+                for (final CompletableFuture> partOf : futureGroup.futures()) {
+                    allResult.addAll(partOf.join());
+                }
+                future.complete(allResult);
+            } else {
+                future.completeExceptionally(throwable);
+            }
+        });
+        return future;
+    }
+
+    public static  CompletableFuture> joinMap(final FutureGroup> futureGroup) {
+        return joinMap(futureGroup, 0);
+    }
+
+    public static  CompletableFuture> joinMap(final FutureGroup> futureGroup, final int size) {
+        return joinMap(futureGroup, size, new CompletableFuture<>());
+    }
+
+    public static  CompletableFuture> joinMap(final FutureGroup> futureGroup, final int size,
+                                                              final CompletableFuture> future) {
+        CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+            if (throwable == null) {
+                final Map allResult = size > 0 ? Maps.newHashMapWithExpectedSize(size) : Maps.newHashMap();
+                for (final CompletableFuture> partOf : futureGroup.futures()) {
+                    allResult.putAll(partOf.join());
+                }
+                future.complete(allResult);
+            } else {
+                future.completeExceptionally(throwable);
+            }
+        });
+        return future;
+    }
+
+    private FutureHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/LoadBalancer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/LoadBalancer.java
new file mode 100644
index 0000000..6377641
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/LoadBalancer.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface LoadBalancer {
+
+    /**
+     * Select one from the element list.
+     */
+     T select(final List elements);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTable.java
new file mode 100644
index 0000000..3342804
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTable.java
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.StampedLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.errors.RouteTableException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * Region routing table.
+ *
+ * Enter a 'key' or a 'key range', which can calculate the region
+ * in which the 'key' is located, and can also calculate all
+ * regions of a 'key range' hit.
+ *
+ * If the pd server is enabled, the routing data will be refreshed
+ * from the pd server, otherwise the routing data is completely
+ * based on the local configuration.
+ *
+ * 
+ *
+ *                                         ┌───────────┐
+ *                                         │ input key │
+ *                                         └─────┬─────┘
+ *                                               │
+ *                                               │
+ *                                               │
+ * ┌ ─ ─ ─ ─ ─ ─ ─ ─ ─        ┌ ─ ─ ─ ─ ─ ─ ┐    │      ┌ ─ ─ ─ ─ ─ ─ ┐           ┌ ─ ─ ─ ─ ─ ─ ┐
+ *  startKey1=byte[0] │          startKey2       │         startKey3                 startKey4
+ * └ ─ ─ ─ ┬ ─ ─ ─ ─ ─        └ ─ ─ ─│─ ─ ─ ┘    │      └ ─ ─ ─│─ ─ ─ ┘           └ ─ ─ ─│─ ─ ─ ┘
+ *         │                         │           │             │                         │
+ *         ▼─────────────────────────▼───────────▼─────────────▼─────────────────────────▼─────────────────────────┐
+ *         │                         │                         │                         │                         │
+ *         │                         │                         │                         │                         │
+ *         │         region1         │         region2         │          region3        │         region4         │
+ *         │                         │                         │                         │                         │
+ *         └─────────────────────────┴─────────────────────────┴─────────────────────────┴─────────────────────────┘
+ *
+ * 

+ *
+ * You can seen that the most suitable data structure for implementing the
+ * above figure is a skip list or a binary tree (for the closest matches for
+ * given search).
+ *
+ * In addition, selecting the startKey or endKey of the region as the key of
+ * the RegionRouteTable is also exquisite.
+ *
+ * For example, why not use endKey?
+ * This depends mainly on the way the region splits:
+ *  a) Suppose that region2[startKey2, endKey2) with id 2 is split
+ *  b) The two regions after splitting are region2[startKey2, splitKey) with
+ *      id continuing to 2 and region3[splitKey, endKey2) with id 3.
+ *  c) At this point, you only need to add an element  to
+ *      the RegionRouteTable. The data of region2 does not need to be modified.
+ *
+ * @author jiachun.fjc
+ */
+public class RegionRouteTable {
+
+    private static final Logger              LOG                = LoggerFactory.getLogger(RegionRouteTable.class);
+
+    private static final Comparator  keyBytesComparator = BytesUtil.getDefaultByteArrayComparator();
+
+    private final StampedLock                stampedLock        = new StampedLock();
+    private final NavigableMap rangeTable         = new TreeMap<>(keyBytesComparator);
+    private final Map          regionTable        = Maps.newHashMap();
+
+    public Region getRegionById(final long regionId) {
+        final StampedLock stampedLock = this.stampedLock;
+        long stamp = stampedLock.tryOptimisticRead();
+        // validate() emit a load-fence, but no store-fence.  So you should only have
+        // load instructions inside a block of tryOptimisticRead() / validate(),
+        // because it is meant to the a read-only operation, and therefore, it is fine
+        // to use the loadFence() function to avoid re-ordering.
+        Region region = safeCopy(this.regionTable.get(regionId));
+        if (!stampedLock.validate(stamp)) {
+            stamp = stampedLock.readLock();
+            try {
+                region = safeCopy(this.regionTable.get(regionId));
+            } finally {
+                stampedLock.unlockRead(stamp);
+            }
+        }
+        return region;
+    }
+
+    public void addOrUpdateRegion(final Region region) {
+        Requires.requireNonNull(region, "region");
+        Requires.requireNonNull(region.getRegionEpoch(), "regionEpoch");
+        final long regionId = region.getId();
+        final byte[] startKey = BytesUtil.nullToEmpty(region.getStartKey());
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.writeLock();
+        try {
+            this.regionTable.put(regionId, region.copy());
+            this.rangeTable.put(startKey, regionId);
+        } finally {
+            stampedLock.unlockWrite(stamp);
+        }
+    }
+
+    public void splitRegion(final long leftId, final Region right) {
+        Requires.requireNonNull(right, "right");
+        Requires.requireNonNull(right.getRegionEpoch(), "right.regionEpoch");
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.writeLock();
+        try {
+            final Region left = this.regionTable.get(leftId);
+            Requires.requireNonNull(left, "left");
+            final byte[] leftStartKey = BytesUtil.nullToEmpty(left.getStartKey());
+            final byte[] leftEndKey = left.getEndKey();
+            final long rightId = right.getId();
+            final byte[] rightStartKey = right.getStartKey();
+            final byte[] rightEndKey = right.getEndKey();
+            Requires.requireNonNull(rightStartKey, "rightStartKey");
+            Requires.requireTrue(BytesUtil.compare(leftStartKey, rightStartKey) < 0,
+                "leftStartKey must < rightStartKey");
+            if (leftEndKey == null || rightEndKey == null) {
+                Requires.requireTrue(leftEndKey == rightEndKey, "leftEndKey must == rightEndKey");
+            } else {
+                Requires.requireTrue(BytesUtil.compare(leftEndKey, rightEndKey) == 0, "leftEndKey must == rightEndKey");
+                Requires.requireTrue(BytesUtil.compare(rightStartKey, rightEndKey) < 0,
+                    "rightStartKey must < rightEndKey");
+            }
+            final RegionEpoch leftEpoch = left.getRegionEpoch();
+            leftEpoch.setVersion(leftEpoch.getVersion() + 1);
+            left.setEndKey(rightStartKey);
+            this.regionTable.put(rightId, right.copy());
+            this.rangeTable.put(rightStartKey, rightId);
+        } finally {
+            stampedLock.unlockWrite(stamp);
+        }
+    }
+
+    public boolean removeRegion(final long regionId) {
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.writeLock();
+        try {
+            final Region region = this.regionTable.remove(regionId);
+            if (region != null) {
+                final byte[] startKey = BytesUtil.nullToEmpty(region.getStartKey());
+                return this.rangeTable.remove(startKey) != null;
+            }
+        } finally {
+            stampedLock.unlockWrite(stamp);
+        }
+        return false;
+    }
+
+    /**
+     * Returns the region to which the key belongs.
+     */
+    public Region findRegionByKey(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.readLock();
+        try {
+            return findRegionByKeyWithoutLock(key);
+        } finally {
+            stampedLock.unlockRead(stamp);
+        }
+    }
+
+    private Region findRegionByKeyWithoutLock(final byte[] key) {
+        // return the greatest key less than or equal to the given key
+        final Map.Entry entry = this.rangeTable.floorEntry(key);
+        if (entry == null) {
+            reportFail(key);
+            throw reject(key, "fail to find region by key");
+        }
+        return this.regionTable.get(entry.getValue());
+    }
+
+    /**
+     * Returns the list of regions to which the keys belongs.
+     */
+    public Map> findRegionsByKeys(final List keys) {
+        Requires.requireNonNull(keys, "keys");
+        final Map> regionMap = Maps.newHashMap();
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.readLock();
+        try {
+            for (final byte[] key : keys) {
+                final Region region = findRegionByKeyWithoutLock(key);
+                regionMap.computeIfAbsent(region, k -> Lists.newArrayList()).add(key);
+            }
+            return regionMap;
+        } finally {
+            stampedLock.unlockRead(stamp);
+        }
+    }
+
+    /**
+     * Returns the list of regions to which the keys belongs.
+     */
+    public Map> findRegionsByKvEntries(final List kvEntries) {
+        Requires.requireNonNull(kvEntries, "kvEntries");
+        final Map> regionMap = Maps.newHashMap();
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.readLock();
+        try {
+            for (final KVEntry kvEntry : kvEntries) {
+                final Region region = findRegionByKeyWithoutLock(kvEntry.getKey());
+                regionMap.computeIfAbsent(region, k -> Lists.newArrayList()).add(kvEntry);
+            }
+            return regionMap;
+        } finally {
+            stampedLock.unlockRead(stamp);
+        }
+    }
+
+    /**
+     * Returns the list of regions to which the keys belongs.
+     */
+    public Map> findRegionsByCASEntries(final List casEntries) {
+        Requires.requireNonNull(casEntries, "casEntries");
+        final Map> regionMap = Maps.newHashMap();
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.readLock();
+        try {
+            for (final CASEntry casEntry : casEntries) {
+                final Region region = findRegionByKeyWithoutLock(casEntry.getKey());
+                regionMap.computeIfAbsent(region, k -> Lists.newArrayList()).add(casEntry);
+            }
+            return regionMap;
+        } finally {
+            stampedLock.unlockRead(stamp);
+        }
+    }
+
+    /**
+     * Returns the list of regions covered by startKey and endKey.
+     */
+    public List findRegionsByKeyRange(final byte[] startKey, final byte[] endKey) {
+        final StampedLock stampedLock = this.stampedLock;
+        final long stamp = stampedLock.readLock();
+        try {
+            final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+            final NavigableMap subRegionMap;
+            if (endKey == null) {
+                subRegionMap = this.rangeTable.tailMap(realStartKey, false);
+            } else {
+                subRegionMap = this.rangeTable.subMap(realStartKey, false, endKey, true);
+            }
+            final List regionList = Lists.newArrayListWithCapacity(subRegionMap.size() + 1);
+            final Map.Entry headEntry = this.rangeTable.floorEntry(realStartKey);
+            if (headEntry == null) {
+                reportFail(startKey);
+                throw reject(startKey, "fail to find region by startKey");
+            }
+            regionList.add(safeCopy(this.regionTable.get(headEntry.getValue())));
+            for (final Long regionId : subRegionMap.values()) {
+                regionList.add(safeCopy(this.regionTable.get(regionId)));
+            }
+            return regionList;
+        } finally {
+            stampedLock.unlockRead(stamp);
+        }
+    }
+
+    /**
+     * Returns the startKey of next region.
+     */
+    public byte[] findStartKeyOfNextRegion(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        final StampedLock stampedLock = this.stampedLock;
+        long stamp = stampedLock.tryOptimisticRead();
+        // get the least key strictly greater than the given key
+        byte[] nextStartKey = this.rangeTable.higherKey(key);
+        if (!stampedLock.validate(stamp)) {
+            stamp = stampedLock.readLock();
+            try {
+                // get the least key strictly greater than the given key
+                nextStartKey = this.rangeTable.higherKey(key);
+            } finally {
+                stampedLock.unlockRead(stamp);
+            }
+        }
+        return nextStartKey;
+    }
+
+    // Should be in lock
+    //
+    // If this method is called, either because the registered region table is incomplete (by user)
+    // or because of a bug.
+    private void reportFail(final byte[] relatedKey) {
+        if (LOG.isErrorEnabled()) {
+            LOG.error("There is a high probability that the data in the region table is corrupted.");
+            LOG.error("---------------------------------------------------------------------------");
+            LOG.error("* RelatedKey:  {}.", BytesUtil.toHex(relatedKey));
+            LOG.error("* RangeTable:  {}.", this.rangeTable);
+            LOG.error("* RegionTable: {}.", this.regionTable);
+            LOG.error("---------------------------------------------------------------------------");
+        }
+    }
+
+    private static Region safeCopy(final Region region) {
+        if (region == null) {
+            return null;
+        }
+        return region.copy();
+    }
+
+    private static RouteTableException reject(final byte[] relatedKey, final String message) {
+        return new RouteTableException("key: " + BytesUtil.toHex(relatedKey) + ", message: " + message);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaIterator.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaIterator.java
new file mode 100644
index 0000000..eb5e931
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaIterator.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.NoSuchElementException;
+
+/**
+ * An iterator over RheaKVStore.
+ *
+ * @param  the type of elements returned by this iterator
+ *
+ * @author jiachun.fjc
+ */
+public interface RheaIterator {
+
+    /**
+     * Returns {@code true} if the iteration has more elements.
+     * (In other words, returns {@code true} if {@link #next} would
+     * return an element rather than throwing an exception.)
+     *
+     * @return {@code true} if the iteration has more elements
+     */
+    boolean hasNext();
+
+    /**
+     * Returns the next element in the iteration.
+     *
+     * @return the next element in the iteration
+     * @throws NoSuchElementException if the iteration has no more elements
+     */
+    E next();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVCliService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVCliService.java
new file mode 100644
index 0000000..3b1543f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVCliService.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.option.CliOptions;
+
+/**
+ * RheaKV client command-line service.
+ *
+ * @author jiachun.fjc
+ */
+public interface RheaKVCliService extends Lifecycle {
+
+    /**
+     * Send a split instruction to the specified region.
+     *
+     * @param regionId    region id
+     * @param newRegionId id of the new region after splitting
+     * @param groupId     the raft group id
+     * @param conf        current configuration
+     * @return operation status
+     */
+    Status rangeSplit(final long regionId, final long newRegionId, final String groupId, final Configuration conf);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVRpcService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVRpcService.java
new file mode 100644
index 0000000..508f882
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVRpcService.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+
+/**
+ * RheaKV's rpc client for sending kv requests and receiving kv responses.
+ *
+ * @author jiachun.fjc
+ */
+public interface RheaKVRpcService extends Lifecycle {
+
+    /**
+     * @see #callAsyncWithRpc(BaseRequest, FailoverClosure, Errors, boolean)
+     */
+     CompletableFuture callAsyncWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                              final Errors lastCause);
+
+    /**
+     * Send KV requests to the remote data service nodes.
+     *
+     * @param request       request data
+     * @param closure       callback for failover strategy
+     * @param lastCause     the exception information held by the last call
+     *                      failed, the initial value is null
+     * @param requireLeader if true, then request to call the leader node
+     * @param            the type of response
+     * @return a future with response
+     */
+     CompletableFuture callAsyncWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                              final Errors lastCause, final boolean requireLeader);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVStore.java
new file mode 100644
index 0000000..7c5b993
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RheaKVStore.java
@@ -0,0 +1,753 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.rhea.FollowerStateListener;
+import com.alipay.sofa.jraft.rhea.LeaderStateListener;
+import com.alipay.sofa.jraft.rhea.StateListener;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * User layer KV store api.
+ *
+ * 
+ *                           ┌────────────────────────────┐
+ *                           │                            │
+ *                           │        RheaKVStore         │────────────────┐
+ *                           │                            │                │
+ *                           └────────────────────────────┘                ▼
+ *                                        │ ▲               ┌────────────────────────────┐
+ *                                        │ │               │   PlacementDriverClient    │
+ *                                        │ │               └────────────────────────────┘
+ *                                        │ │                              │
+ *                                        │ │                              ▼
+ *                                        │ │               ┌────────────────────────────┐
+ *                                        │ │               │      RegionRouteTable      │
+ *                                        │ │               └────────────────────────────┘
+ *                    ┌───────────────────┘ │                              │
+ *                    │                     │                              ▼
+ *                    │                     │               ┌────────────────────────────┐
+ *                    │                     └───────────────│        LoadBalancer        │
+ *                  split                                   └────────────────────────────┘
+ *                 request                             local
+ *                    ├────────────────────────────────invoke──────────────────────────────────────────┐
+ *                    │                                                                                │
+ *                    ▼                                                                                │
+ *     ┌────────────────────────────┐           ┌────────────────────────────┐                         │
+ *     │      RheaKVRpcService      │───rpc────▶│     KVCommandProcessor     │                         │
+ *     └────────────────────────────┘           └────────────────────────────┘                         │
+ *                                                             │                                       │
+ *                                                             ▼                                       ▼
+ *                                              ┌────────────────────────────┐          ┌────────────────────────────┐
+ *                                              │      RegionKVService       │────────▶ │     MetricsRawKVStore      │
+ *                                              └────────────────────────────┘          └────────────────────────────┘
+ *                                                                                                     │
+ *                                                                                                     │
+ *                                                                                                     ▼
+ *     ┌────────────────────────────┐           ┌────────────────────────────┐          ┌────────────────────────────┐
+ *     │      RocksRawKVStore       │◀──────────│    KVStoreStateMachine     │◀──raft───│       RaftRawKVStore       │
+ *     └────────────────────────────┘           └────────────────────────────┘          └────────────────────────────┘
+ * 

+ *
+ * @author jiachun.fjc
+ */
+public interface RheaKVStore extends Lifecycle {
+
+    /**
+     * Equivalent to {@code get(key, true)}.
+     */
+    CompletableFuture get(final byte[] key);
+
+    /**
+     * @see #get(byte[])
+     */
+    CompletableFuture get(final String key);
+
+    /**
+     * Get which returns a new byte array storing the value associated
+     * with the specified input key if any.  null will be returned if
+     * the specified key is not found.
+     *
+     * @param key          the key retrieve the value.
+     * @param readOnlySafe provide consistent reading if {@code readOnlySafe}
+     *                     is true.
+     * @return a byte array storing the value associated with the input key if
+     * any.  null if it does not find the specified key.
+     */
+    CompletableFuture get(final byte[] key, final boolean readOnlySafe);
+
+    /**
+     * @see #get(byte[], boolean)
+     */
+    CompletableFuture get(final String key, final boolean readOnlySafe);
+
+    /**
+     * @see #get(byte[])
+     */
+    byte[] bGet(final byte[] key);
+
+    /**
+     * @see #get(String)
+     */
+    byte[] bGet(final String key);
+
+    /**
+     * @see #get(byte[], boolean)
+     */
+    byte[] bGet(final byte[] key, final boolean readOnlySafe);
+
+    /**
+     * @see #get(String, boolean)
+     */
+    byte[] bGet(final String key, final boolean readOnlySafe);
+
+    /**
+     * Equivalent to {@code multiGet(keys, true)}.
+     */
+    CompletableFuture> multiGet(final List keys);
+
+    /**
+     * Returns a map of keys for which values were found in database.
+     *
+     * @param keys         list of keys for which values need to be retrieved.
+     * @param readOnlySafe provide consistent reading if {@code readOnlySafe}
+     *                     is true.
+     * @return a map where key of map is the key passed by user and value for map
+     * entry is the corresponding value in database.
+     */
+    CompletableFuture> multiGet(final List keys, final boolean readOnlySafe);
+
+    /**
+     * @see #multiGet(List)
+     */
+    Map bMultiGet(final List keys);
+
+    /**
+     * @see #multiGet(List, boolean)
+     */
+    Map bMultiGet(final List keys, final boolean readOnlySafe);
+
+    /**
+     * Returns whether database contains the specified input key.
+     *
+     * @param key the specified key database contains.
+     * @return whether database contains the specified key.
+     */
+    CompletableFuture containsKey(final byte[] key);
+
+    /**
+     * @see #containsKey(byte[])
+     */
+    CompletableFuture containsKey(final String key);
+
+    /**
+     * @see #containsKey(byte[])
+     */
+    Boolean bContainsKey(final byte[] key);
+
+    /**
+     * @see #containsKey(byte[])
+     */
+    Boolean bContainsKey(final String key);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, true)}.
+     */
+    CompletableFuture> scan(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #scan(byte[], byte[])
+     */
+    CompletableFuture> scan(final String startKey, final String endKey);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, readOnlySafe, true)}.
+     */
+    CompletableFuture> scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #scan(byte[], byte[], boolean)
+     */
+    CompletableFuture> scan(final String startKey, final String endKey, final boolean readOnlySafe);
+
+    /**
+     * Query all data in the key of range [startKey, endKey).
+     * 

+     * Provide consistent reading if {@code readOnlySafe} is true.
+     *
+     * Scanning across multi regions maybe slower and devastating.
+     *
+     * @param startKey     first key to scan within database (included),
+     *                     null means 'min-key' in the database.
+     * @param endKey       last key to scan within database (excluded).
+     *                     null means 'max-key' in the database.
+     * @param readOnlySafe provide consistent reading if {@code readOnlySafe}
+     *                     is true.
+     * @param returnValue  whether to return value.
+     * @return a list where the key of range [startKey, endKey) passed by user
+     * and value for {@code KVEntry}
+     */
+    CompletableFuture> scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                                          final boolean returnValue);
+
+    /**
+     * @see #scan(byte[], byte[], boolean, boolean)
+     */
+    CompletableFuture> scan(final String startKey, final String endKey, final boolean readOnlySafe,
+                                          final boolean returnValue);
+
+    /**
+     * @see #scan(byte[], byte[])
+     */
+    List bScan(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #scan(String, String)
+     */
+    List bScan(final String startKey, final String endKey);
+
+    /**
+     * @see #scan(String, String, boolean)
+     */
+    List bScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #scan(String, String, boolean)
+     */
+    List bScan(final String startKey, final String endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #scan(String, String, boolean, boolean)
+     */
+    List bScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                        final boolean returnValue);
+
+    /**
+     * @see #scan(String, String, boolean, boolean)
+     */
+    List bScan(final String startKey, final String endKey, final boolean readOnlySafe,
+                        final boolean returnValue);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, true)}.
+     */
+    CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #reverseScan(byte[], byte[])
+     */
+    CompletableFuture> reverseScan(final String startKey, final String endKey);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, readOnlySafe, true)}.
+     */
+    CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #reverseScan(byte[], byte[], boolean)
+     */
+    CompletableFuture> reverseScan(final String startKey, final String endKey, final boolean readOnlySafe);
+
+    /**
+     * Reverse query all data in the key of range [startKey, endKey).
+     * 

+     * Provide consistent reading if {@code readOnlySafe} is true.
+     *
+     * Reverse scanning is usually much worse than forward scanning.
+     *
+     * Reverse scanning across multi regions maybe slower and devastating.
+     *
+     * @param startKey     first key to reverse scan within database (included),
+     *                     null means 'max-key' in the database.
+     * @param endKey       last key to reverse scan within database (excluded).
+     *                     null means 'min-key' in the database.
+     * @param readOnlySafe provide consistent reading if {@code readOnlySafe}
+     *                     is true.
+     * @param returnValue  whether to return value.
+     * @return a list where the key of range [startKey, endKey) passed by user
+     * and value for {@code KVEntry}
+     */
+    CompletableFuture> reverseScan(final byte[] startKey, final byte[] endKey,
+                                                 final boolean readOnlySafe, final boolean returnValue);
+
+    /**
+     * @see #reverseScan(byte[], byte[], boolean, boolean)
+     */
+    CompletableFuture> reverseScan(final String startKey, final String endKey,
+                                                 final boolean readOnlySafe, final boolean returnValue);
+
+    /**
+     * @see #reverseScan(byte[], byte[])
+     */
+    List bReverseScan(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #scan(String, String)
+     */
+    List bReverseScan(final String startKey, final String endKey);
+
+    /**
+     * @see #scan(String, String, boolean)
+     */
+    List bReverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #scan(String, String, boolean)
+     */
+    List bReverseScan(final String startKey, final String endKey, final boolean readOnlySafe);
+
+    /**
+     * @see #reverseScan(String, String, boolean, boolean)
+     */
+    List bReverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                               final boolean returnValue);
+
+    /**
+     * @see #reverseScan(String, String, boolean, boolean)
+     */
+    List bReverseScan(final String startKey, final String endKey, final boolean readOnlySafe,
+                               final boolean returnValue);
+
+    /**
+     * Equivalent to {@code iterator(startKey, endKey, bufSize, true)}.
+     */
+    RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize);
+
+    /**
+     * @see #iterator(byte[], byte[], int)
+     */
+    RheaIterator iterator(final String startKey, final String endKey, final int bufSize);
+
+    /**
+     * Equivalent to {@code iterator(startKey, endKey, bufSize, true, true)}.
+     */
+    RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize,
+                                   final boolean readOnlySafe);
+
+    /**
+     * @see #iterator(byte[], byte[], int, boolean)
+     */
+    RheaIterator iterator(final String startKey, final String endKey, final int bufSize,
+                                   final boolean readOnlySafe);
+
+    /**
+     * Returns a remote iterator over the contents of the database.
+     *
+     * Functionally similar to {@link #scan(byte[], byte[], boolean)},
+     * but iterator only returns a small amount of data at a time, avoiding
+     * a large amount of data returning to the client at one time causing
+     * memory overflow, can think of it as a 'lazy scan' method.
+     *
+     * @param startKey     first key to scan within database (included),
+     *                     null means 'min-key' in the database.
+     * @param endKey       last key to scan within database (excluded),
+     *                     null means 'max-key' in the database.
+     * @param readOnlySafe provide consistent reading if {@code readOnlySafe}
+     *                     is true.
+     * @param returnValue  whether to return value.
+     * @return a iterator where the key of range [startKey, endKey) passed by
+     * user and value for {@code KVEntry}
+     */
+    RheaIterator iterator(final byte[] startKey, final byte[] endKey, final int bufSize,
+                                   final boolean readOnlySafe, final boolean returnValue);
+
+    /**
+     * @see #iterator(byte[], byte[], int, boolean, boolean)
+     */
+    RheaIterator iterator(final String startKey, final String endKey, final int bufSize,
+                                   final boolean readOnlySafe, final boolean returnValue);
+
+    /**
+     * Get a globally unique auto-increment sequence.
+     *
+     * Be careful do not to try to get or update the value of {@code seqKey}
+     * by other methods, you won't get it.
+     *
+     * @param seqKey the key of sequence
+     * @param step   number of values obtained
+     * @return a values range of [startValue, endValue)
+     */
+    CompletableFuture getSequence(final byte[] seqKey, final int step);
+
+    /**
+     * @see #getSequence(byte[], int)
+     */
+    CompletableFuture getSequence(final String seqKey, final int step);
+
+    /**
+     * @see #getSequence(byte[], int)
+     */
+    Sequence bGetSequence(final byte[] seqKey, final int step);
+
+    /**
+     * @see #getSequence(byte[], int)
+     */
+    Sequence bGetSequence(final String seqKey, final int step);
+
+    /**
+     * Gets the latest sequence start value, this is a read-only operation.
+     *
+     * Equivalent to {@code getSequence(seqKey, 0)}.
+     *
+     * @see #getSequence(byte[], int)
+     *
+     * @param seqKey the key of sequence
+     * @return the latest sequence value
+     */
+    CompletableFuture getLatestSequence(final byte[] seqKey);
+
+    /**
+     * @see #getLatestSequence(byte[])
+     */
+    CompletableFuture getLatestSequence(final String seqKey);
+
+    /**
+     * @see #getLatestSequence(byte[])
+     */
+    Long bGetLatestSequence(final byte[] seqKey);
+
+    /**
+     * @see #getLatestSequence(byte[])
+     */
+    Long bGetLatestSequence(final String seqKey);
+
+    /**
+     * Reset the sequence to 0.
+     *
+     * @param seqKey the key of sequence
+     */
+    CompletableFuture resetSequence(final byte[] seqKey);
+
+    /**
+     * @see #resetSequence(byte[])
+     */
+    CompletableFuture resetSequence(final String seqKey);
+
+    /**
+     * @see #resetSequence(byte[])
+     */
+    Boolean bResetSequence(final byte[] seqKey);
+
+    /**
+     * @see #resetSequence(byte[])
+     */
+    Boolean bResetSequence(final String seqKey);
+
+    /**
+     * Set the database entry for "key" to "value".
+     *
+     * @param key   the specified key to be inserted.
+     * @param value the value associated with the specified key.
+     * @return {@code true} if success.
+     */
+    CompletableFuture put(final byte[] key, final byte[] value);
+
+    /**
+     * @see #put(byte[], byte[])
+     */
+    CompletableFuture put(final String key, final byte[] value);
+
+    /**
+     * @see #put(byte[], byte[])
+     */
+    Boolean bPut(final byte[] key, final byte[] value);
+
+    /**
+     * @see #put(byte[], byte[])
+     */
+    Boolean bPut(final String key, final byte[] value);
+
+    /**
+     * Set the database entry for "key" to "value", and return the
+     * previous value associated with "key", or null if there was no
+     * mapping for "key".
+     * @param key   the specified key to be inserted.
+     * @param value the value associated with the specified key.
+     * @return the previous value associated with "key", or null if
+     * there was no mapping for "key".
+     */
+    CompletableFuture getAndPut(final byte[] key, final byte[] value);
+
+    /**
+     * @see #getAndPut(byte[], byte[])
+     */
+    CompletableFuture getAndPut(final String key, final byte[] value);
+
+    /**
+     * @see #getAndPut(byte[], byte[])
+     */
+    byte[] bGetAndPut(final byte[] key, final byte[] value);
+
+    /**
+     * @see #getAndPut(byte[], byte[])
+     */
+    byte[] bGetAndPut(final String key, final byte[] value);
+
+    /**
+     * Atomically sets the value to the given updated value
+     * if the current value equal (compare bytes) the expected value.
+     *
+     * @param key    the key retrieve the value
+     * @param expect the expected value
+     * @param update the new value
+     * @return true if successful. False return indicates that the actual
+     * value was not equal to the expected value.
+     */
+    CompletableFuture compareAndPut(final byte[] key, final byte[] expect, final byte[] update);
+
+    /**
+     * @see #compareAndPut(byte[], byte[], byte[])
+     */
+    CompletableFuture compareAndPut(final String key, final byte[] expect, final byte[] update);
+
+    /**
+     * @see #compareAndPut(byte[], byte[], byte[])
+     */
+    Boolean bCompareAndPut(final byte[] key, final byte[] expect, final byte[] update);
+
+    /**
+     * @see #compareAndPut(byte[], byte[], byte[])
+     */
+    Boolean bCompareAndPut(final String key, final byte[] expect, final byte[] update);
+
+    /**
+     * Add merge operand for key/value pair.
+     *
+     * 
+     *     // Writing aa under key
+     *     db.put("key", "aa");
+     *
+     *     // Writing bb under key
+     *     db.merge("key", "bb");
+     *
+     *     assertThat(db.get("key")).isEqualTo("aa,bb");
+     * 

+     *
+     * @param key   the specified key to be merged.
+     * @param value the value to be merged with the current value for
+     *              the specified key.
+     * @return {@code true} if success.
+     */
+    CompletableFuture merge(final String key, final String value);
+
+    /**
+     * @see #merge(String, String)
+     */
+    Boolean bMerge(final String key, final String value);
+
+    /**
+     * The batch method of {@link #put(byte[], byte[])}
+     */
+    CompletableFuture put(final List entries);
+
+    /**
+     * @see #put(List)
+     */
+    Boolean bPut(final List entries);
+
+    /**
+     * The batch method of {@link #compareAndPut(byte[], byte[], byte[])}
+     */
+    CompletableFuture compareAndPutAll(final List entries);
+
+    /**
+     * @see #compareAndPutAll(List)
+     */
+    Boolean bCompareAndPutAll(final List entries);
+
+    /**
+     * If the specified key is not already associated with a value
+     * associates it with the given value and returns {@code null},
+     * else returns the current value.
+     *
+     * @param key   the specified key to be inserted.
+     * @param value the value associated with the specified key.
+     * @return the previous value associated with the specified key,
+     * or {@code null} if there was no mapping for the key.
+     * (A {@code null} return can also indicate that the database.
+     * previously associated {@code null} with the key.
+     */
+    CompletableFuture putIfAbsent(final byte[] key, final byte[] value);
+
+    /**
+     * @see #putIfAbsent(byte[], byte[])
+     */
+    CompletableFuture putIfAbsent(final String key, final byte[] value);
+
+    /**
+     * @see #putIfAbsent(byte[], byte[])
+     */
+    byte[] bPutIfAbsent(final byte[] key, final byte[] value);
+
+    /**
+     * @see #putIfAbsent(byte[], byte[])
+     */
+    byte[] bPutIfAbsent(final String key, final byte[] value);
+
+    /**
+     * Delete the database entry (if any) for "key".
+     *
+     * @param key key to delete within database.
+     * @return {@code true} if success.
+     */
+    CompletableFuture delete(final byte[] key);
+
+    /**
+     * @see #delete(byte[])
+     */
+    CompletableFuture delete(final String key);
+
+    /**
+     * @see #delete(byte[])
+     */
+    Boolean bDelete(final byte[] key);
+
+    /**
+     * @see #delete(byte[])
+     */
+    Boolean bDelete(final String key);
+
+    /**
+     * Removes the database entries in the range ["startKey", "endKey"), i.e.,
+     * including "startKey" and excluding "endKey".
+     *
+     * @param startKey first key to delete within database (included)
+     * @param endKey   last key to delete within database (excluded)
+     * @return {@code true} if success.
+     */
+    CompletableFuture deleteRange(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #deleteRange(byte[], byte[])
+     */
+    CompletableFuture deleteRange(final String startKey, final String endKey);
+
+    /**
+     * @see #deleteRange(byte[], byte[])
+     */
+    Boolean bDeleteRange(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * @see #deleteRange(byte[], byte[])
+     */
+    Boolean bDeleteRange(final String startKey, final String endKey);
+
+    /**
+     * The batch method of {@link #delete(byte[])}
+     */
+    CompletableFuture delete(final List keys);
+
+    /**
+     * @see #delete(List)
+     */
+    Boolean bDelete(final List keys);
+
+    /**
+     * @see #getDistributedLock(byte[], long, TimeUnit, ScheduledExecutorService)
+     */
+    DistributedLock getDistributedLock(final byte[] target, final long lease, final TimeUnit unit);
+
+    /**
+     * @see #getDistributedLock(String, long, TimeUnit, ScheduledExecutorService)
+     */
+    DistributedLock getDistributedLock(final String target, final long lease, final TimeUnit unit);
+
+    /**
+     * Creates a distributed lock implementation that provides
+     * exclusive access to a shared resource.
+     * 

+     * 
+     *      DistributedLock lock = ...;
+     *      if (lock.tryLock()) {
+     *          try {
+     *              // manipulate protected state
+     *          } finally {
+     *              lock.unlock();
+     *          }
+     *      } else {
+     *          // perform alternative actions
+     *      }
+     * 

+     *
+     * The algorithm relies on the assumption that while there is no
+     * synchronized clock across the processes, still the local time in
+     * every process flows approximately at the same rate, with an error
+     * which is small compared to the auto-release time of the lock.
+     *
+     * @param target   key of the distributed lock that acquired.
+     * @param lease    the lease time for the distributed lock to live.
+     * @param unit     the time unit of the {@code expire} argument.
+     * @param watchdog if the watchdog is not null, it will auto keep
+     *                 lease of current lock, otherwise won't keep lease,
+     *                 this method dose not pay attention to the life cycle
+     *                 of watchdog, please maintain it yourself.
+     * @return a distributed lock instance.
+     */
+    DistributedLock getDistributedLock(final byte[] target, final long lease, final TimeUnit unit,
+                                               final ScheduledExecutorService watchdog);
+
+    /**
+     * @see #getDistributedLock(byte[], long, TimeUnit, ScheduledExecutorService)
+     */
+    DistributedLock getDistributedLock(final String target, final long lease, final TimeUnit unit,
+                                               final ScheduledExecutorService watchdog);
+
+    /**
+     * Returns current placement driver client instance.
+     */
+    PlacementDriverClient getPlacementDriverClient();
+
+    /**
+     * Add a listener for the state change of the leader with
+     * this region.
+     * 

+     * In a special case, if that is a single region environment,
+     * then regionId = -1 as the input parameter.
+     */
+    void addLeaderStateListener(final long regionId, final LeaderStateListener listener);
+
+    /**
+     * Add a listener for the state change of the follower with
+     * this region.
+     * 

+     * In a special case, if that is a single region environment,
+     * then regionId = -1 as the input parameter.
+     */
+    void addFollowerStateListener(final long regionId, final FollowerStateListener listener);
+
+    /**
+     * Add a listener for the state change (leader, follower) with
+     * this region.
+     * 

+     * In a special case, if that is a single region environment,
+     * then regionId = -1 as the input parameter.
+     */
+    void addStateListener(final long regionId, final StateListener listener);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancer.java
new file mode 100644
index 0000000..2fa1931
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancer.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RoundRobinLoadBalancer implements LoadBalancer {
+
+    private static final ConcurrentMap       container    = Maps.newConcurrentMapLong();
+
+    private static final AtomicIntegerFieldUpdater indexUpdater = AtomicIntegerFieldUpdater
+                                                                                            .newUpdater(
+                                                                                                RoundRobinLoadBalancer.class,
+                                                                                                "index");
+
+    @SuppressWarnings("unused")
+    private volatile int                                                   index        = 0;
+
+    public static RoundRobinLoadBalancer getInstance(final long regionId) {
+        RoundRobinLoadBalancer instance = container.get(regionId);
+        if (instance == null) {
+            RoundRobinLoadBalancer newInstance = new RoundRobinLoadBalancer();
+            instance = container.putIfAbsent(regionId, newInstance);
+            if (instance == null) {
+                instance = newInstance;
+            }
+        }
+        return instance;
+    }
+
+    @Override
+    public  T select(final List elements) {
+        if (elements == null) {
+            throw new NullPointerException("elements");
+        }
+
+        final int size = elements.size();
+
+        if (size == 1) {
+            return elements.get(0);
+        }
+
+        final int roundRobinIndex = indexUpdater.getAndIncrement(this) & Integer.MAX_VALUE;
+
+        return elements.get(roundRobinIndex % size);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/FailoverClosure.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/FailoverClosure.java
new file mode 100644
index 0000000..4b2ab57
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/FailoverClosure.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreClosure;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface FailoverClosure extends KVStoreClosure {
+
+    CompletableFuture future();
+
+    void success(final T result);
+
+    void failure(final Throwable cause);
+
+    void failure(final Errors error);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/ListRetryCallable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/ListRetryCallable.java
new file mode 100644
index 0000000..43d62da
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/ListRetryCallable.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+
+/**
+ * A retry task that returns a {@link FutureGroup} result.
+ *
+ * @author jiachun.fjc
+ */
+public interface ListRetryCallable {
+
+    FutureGroup> run(final Throwable retryCause);
+}
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryCallable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryCallable.java
new file mode 100644
index 0000000..4741473
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryCallable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover;
+
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+
+/**
+ * A retry task that returns a {@link FutureGroup} result.
+ *
+ * @author jiachun.fjc
+ */
+public interface RetryCallable {
+
+    FutureGroup run(final Throwable retryCause);
+}
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryRunner.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryRunner.java
new file mode 100644
index 0000000..45e3e0f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/RetryRunner.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover;
+
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+
+/**
+ * A retry task.
+ *
+ * @author jiachun.fjc
+ */
+public interface RetryRunner {
+
+    void run(final Errors retryCause);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/BoolFailoverFuture.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/BoolFailoverFuture.java
new file mode 100644
index 0000000..b8f7c0b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/BoolFailoverFuture.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover.impl;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryCallable;
+import com.alipay.sofa.jraft.rhea.errors.ApiExceptionHelper;
+import com.alipay.sofa.jraft.rhea.util.Attachable;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+
+/**
+ * A helper future for bool result failover, which is an immutable object.
+ * A new object will be created when a retry operation occurs and
+ * {@code retriesLeft} will decrease by 1, until {@code retriesLeft} == 0.
+ *
+ * @author jiachun.fjc
+ */
+public final class BoolFailoverFuture extends CompletableFuture implements Attachable {
+
+    private static final Logger          LOG = LoggerFactory.getLogger(BoolFailoverFuture.class);
+
+    private final int                    retriesLeft;
+    private final RetryCallable retryCallable;
+    private final Object                 attachments;
+
+    public BoolFailoverFuture(int retriesLeft, RetryCallable retryCallable) {
+        this(retriesLeft, retryCallable, null);
+    }
+
+    public BoolFailoverFuture(int retriesLeft, RetryCallable retryCallable, Object attachments) {
+        this.retriesLeft = retriesLeft;
+        this.retryCallable = retryCallable;
+        this.attachments = attachments;
+    }
+
+    @Override
+    public boolean completeExceptionally(final Throwable ex) {
+        if (this.retriesLeft > 0 && ApiExceptionHelper.isInvalidEpoch(ex)) {
+            LOG.warn("[InvalidEpoch-Failover] cause: {}, [{}] retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+            final FutureGroup futureGroup = this.retryCallable.run(ex);
+            CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+                if (throwable == null) {
+                    for (final CompletableFuture partOf : futureGroup.futures()) {
+                        if (!partOf.join()) {
+                            super.complete(false);
+                            return;
+                        }
+                    }
+                    super.complete(true);
+                } else {
+                    super.completeExceptionally(throwable);
+                }
+            });
+            return false;
+        }
+        if (this.retriesLeft <= 0) {
+            LOG.error("[InvalidEpoch-Failover] cause: {}, {} retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+        }
+        return super.completeExceptionally(ex);
+    }
+
+    @Override
+    public Object getAttachments() {
+        return attachments;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/FailoverClosureImpl.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/FailoverClosureImpl.java
new file mode 100644
index 0000000..3c80c34
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/FailoverClosureImpl.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover.impl;
+
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryRunner;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.ErrorsHelper;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+
+/**
+ * A helper closure for failover, which is an immutable object.
+ * A new object will be created when a retry operation occurs
+ * and {@code retriesLeft} will decrease by 1, until
+ * {@code retriesLeft} == 0.
+ *
+ * @author jiachun.fjc
+ */
+public final class FailoverClosureImpl extends BaseKVStoreClosure implements FailoverClosure {
+
+    private static final Logger        LOG = LoggerFactory.getLogger(FailoverClosureImpl.class);
+
+    private final CompletableFuture future;
+    private final boolean              retryOnInvalidEpoch;
+    private final int                  retriesLeft;
+    private final RetryRunner          retryRunner;
+
+    public FailoverClosureImpl(CompletableFuture future, int retriesLeft, RetryRunner retryRunner) {
+        this(future, true, retriesLeft, retryRunner);
+    }
+
+    public FailoverClosureImpl(CompletableFuture future, boolean retryOnInvalidEpoch, int retriesLeft,
+                               RetryRunner retryRunner) {
+        this.future = future;
+        this.retryOnInvalidEpoch = retryOnInvalidEpoch;
+        this.retriesLeft = retriesLeft;
+        this.retryRunner = retryRunner;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void run(final Status status) {
+        if (status.isOk()) {
+            success((T) getData());
+            return;
+        }
+
+        final Errors error = getError();
+        if (this.retriesLeft > 0
+            && (ErrorsHelper.isInvalidPeer(error) || (this.retryOnInvalidEpoch && ErrorsHelper.isInvalidEpoch(error)))) {
+            LOG.warn("[Failover] status: {}, error: {}, [{}] retries left.", status, error, this.retriesLeft);
+            this.retryRunner.run(error);
+        } else {
+            if (this.retriesLeft <= 0) {
+                LOG.error("[InvalidEpoch-Failover] status: {}, error: {}, {} retries left.", status, error,
+                    this.retriesLeft);
+            }
+            failure(error);
+        }
+    }
+
+    @Override
+    public CompletableFuture future() {
+        return future;
+    }
+
+    @Override
+    public void success(final T result) {
+        this.future.complete(result);
+    }
+
+    @Override
+    public void failure(final Throwable cause) {
+        this.future.completeExceptionally(cause);
+    }
+
+    @Override
+    public void failure(final Errors error) {
+        if (error == null) {
+            failure(new NullPointerException(
+                "The error message is missing, this should not happen, now only the stack information can be referenced."));
+        } else {
+            failure(error.exception());
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/ListFailoverFuture.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/ListFailoverFuture.java
new file mode 100644
index 0000000..b74a93e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/ListFailoverFuture.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover.impl;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+import com.alipay.sofa.jraft.rhea.client.failover.ListRetryCallable;
+import com.alipay.sofa.jraft.rhea.errors.ApiExceptionHelper;
+import com.alipay.sofa.jraft.rhea.util.Attachable;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+
+/**
+ * A helper future for list result failover, which is an immutable object.
+ * A new object will be created when a retry operation occurs and
+ * {@code retriesLeft} will decrease by 1, until {@code retriesLeft} == 0.
+ *
+ * @author jiachun.fjc
+ */
+public final class ListFailoverFuture extends CompletableFuture> implements Attachable {
+
+    private static final Logger        LOG = LoggerFactory.getLogger(ListFailoverFuture.class);
+
+    private final int                  retriesLeft;
+    private final ListRetryCallable retryCallable;
+    private final Object               attachments;
+
+    public ListFailoverFuture(int retriesLeft, ListRetryCallable retryCallable) {
+        this(retriesLeft, retryCallable, null);
+    }
+
+    public ListFailoverFuture(int retriesLeft, ListRetryCallable retryCallable, Object attachments) {
+        this.retriesLeft = retriesLeft;
+        this.retryCallable = retryCallable;
+        this.attachments = attachments;
+    }
+
+    @Override
+    public boolean completeExceptionally(final Throwable ex) {
+        if (this.retriesLeft > 0 && ApiExceptionHelper.isInvalidEpoch(ex)) {
+            LOG.warn("[InvalidEpoch-Failover] cause: {}, [{}] retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+            final FutureGroup> futureGroup = this.retryCallable.run(ex);
+            CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+                if (throwable == null) {
+                    final List all = Lists.newArrayList();
+                    for (final CompletableFuture> partOf : futureGroup.futures()) {
+                        all.addAll(partOf.join());
+                    }
+                    super.complete(all);
+                } else {
+                    super.completeExceptionally(throwable);
+                }
+            });
+            return false;
+        }
+        if (this.retriesLeft <= 0) {
+            LOG.error("[InvalidEpoch-Failover] cause: {}, {} retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+        }
+        return super.completeExceptionally(ex);
+    }
+
+    @Override
+    public Object getAttachments() {
+        return attachments;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/MapFailoverFuture.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/MapFailoverFuture.java
new file mode 100644
index 0000000..8bc03cb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/failover/impl/MapFailoverFuture.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.failover.impl;
+
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryCallable;
+import com.alipay.sofa.jraft.rhea.errors.ApiExceptionHelper;
+import com.alipay.sofa.jraft.rhea.util.Attachable;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+
+/**
+ * A helper object for map result failover, which is an immutable object.
+ * A new object will be created when a retry operation occurs and
+ * {@code retriesLeft} will decrease by 1, until {@code retriesLeft} == 0.
+ *
+ * @author jiachun.fjc
+ */
+public final class MapFailoverFuture extends CompletableFuture> implements Attachable {
+
+    private static final Logger            LOG = LoggerFactory.getLogger(MapFailoverFuture.class);
+
+    private final int                      retriesLeft;
+    private final RetryCallable> retryCallable;
+    private final Object                   attachments;
+
+    public MapFailoverFuture(int retriesLeft, RetryCallable> retryCallable) {
+        this(retriesLeft, retryCallable, null);
+    }
+
+    public MapFailoverFuture(int retriesLeft, RetryCallable> retryCallable, Object attachments) {
+        this.retriesLeft = retriesLeft;
+        this.retryCallable = retryCallable;
+        this.attachments = attachments;
+    }
+
+    @Override
+    public boolean completeExceptionally(final Throwable ex) {
+        if (this.retriesLeft > 0 && ApiExceptionHelper.isInvalidEpoch(ex)) {
+            LOG.warn("[InvalidEpoch-Failover] cause: {}, [{}] retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+            final FutureGroup> futureGroup = this.retryCallable.run(ex);
+            CompletableFuture.allOf(futureGroup.toArray()).whenComplete((ignored, throwable) -> {
+                if (throwable == null) {
+                    final Map all = Maps.newHashMap();
+                    for (final CompletableFuture> partOf : futureGroup.futures()) {
+                        all.putAll(partOf.join());
+                    }
+                    super.complete(all);
+                } else {
+                    super.completeExceptionally(throwable);
+                }
+            });
+            return false;
+        }
+        if (this.retriesLeft <= 0) {
+            LOG.error("[InvalidEpoch-Failover] cause: {}, {} retries left.", StackTraceUtil.stackTrace(ex),
+                    this.retriesLeft);
+        }
+        return super.completeExceptionally(ex);
+    }
+
+    @Override
+    public Object getAttachments() {
+        return attachments;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/AbstractPlacementDriverClient.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/AbstractPlacementDriverClient.java
new file mode 100644
index 0000000..6d65974
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/AbstractPlacementDriverClient.java
@@ -0,0 +1,440 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.CliService;
+import com.alipay.sofa.jraft.RaftServiceFactory;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.core.CliServiceImpl;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.client.RegionRouteTable;
+import com.alipay.sofa.jraft.rhea.client.RoundRobinLoadBalancer;
+import com.alipay.sofa.jraft.rhea.errors.RouteTableException;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionRouteTableOptions;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.RpcOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.rpc.CliClientService;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.AbstractClientService;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class AbstractPlacementDriverClient implements PlacementDriverClient {
+
+    private static final Logger         LOG                      = LoggerFactory
+                                                                     .getLogger(AbstractPlacementDriverClient.class);
+
+    private static final long           AT_LEAST_REQUIRED_MILLIS = TimeUnit.SECONDS.toMillis(1);
+
+    protected final RegionRouteTable    regionRouteTable         = new RegionRouteTable();
+    protected final long                clusterId;
+    protected final String              clusterName;
+
+    protected CliService                cliService;
+    protected CliClientService          cliClientService;
+    protected RpcClient                 rpcClient;
+    protected PlacementDriverRpcService pdRpcService;
+
+    protected AbstractPlacementDriverClient(long clusterId, String clusterName) {
+        this.clusterId = clusterId;
+        this.clusterName = clusterName;
+    }
+
+    @Override
+    public synchronized boolean init(final PlacementDriverOptions opts) {
+        initCli(opts.getCliOptions());
+        this.pdRpcService = new DefaultPlacementDriverRpcService(this);
+        RpcOptions rpcOpts = opts.getPdRpcOptions();
+        if (rpcOpts == null) {
+            rpcOpts = RpcOptionsConfigured.newDefaultConfig();
+            rpcOpts.setCallbackExecutorCorePoolSize(0);
+            rpcOpts.setCallbackExecutorMaximumPoolSize(0);
+        }
+        if (!this.pdRpcService.init(rpcOpts)) {
+            LOG.error("Fail to init [PlacementDriverRpcService].");
+            return false;
+        }
+        // region route table
+        final List regionRouteTableOptionsList = opts.getRegionRouteTableOptionsList();
+        if (regionRouteTableOptionsList != null) {
+            final String initialServerList = opts.getInitialServerList();
+            for (final RegionRouteTableOptions regionRouteTableOpts : regionRouteTableOptionsList) {
+                if (Strings.isBlank(regionRouteTableOpts.getInitialServerList())) {
+                    // if blank, extends parent's value
+                    regionRouteTableOpts.setInitialServerList(initialServerList);
+                }
+                initRouteTableByRegion(regionRouteTableOpts);
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (this.cliService != null) {
+            this.cliService.shutdown();
+        }
+        if (this.pdRpcService != null) {
+            this.pdRpcService.shutdown();
+        }
+    }
+
+    @Override
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    @Override
+    public Region getRegionById(final long regionId) {
+        return this.regionRouteTable.getRegionById(regionId);
+    }
+
+    @Override
+    public Region findRegionByKey(final byte[] key, final boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findRegionByKey(key);
+    }
+
+    @Override
+    public Map> findRegionsByKeys(final List keys, final boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findRegionsByKeys(keys);
+    }
+
+    @Override
+    public Map> findRegionsByKvEntries(final List kvEntries, final boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findRegionsByKvEntries(kvEntries);
+    }
+
+    @Override
+    public Map> findRegionsByCASEntries(List casEntries, boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findRegionsByCASEntries(casEntries);
+    }
+
+    @Override
+    public List findRegionsByKeyRange(final byte[] startKey, final byte[] endKey, final boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findRegionsByKeyRange(startKey, endKey);
+    }
+
+    @Override
+    public byte[] findStartKeyOfNextRegion(final byte[] key, final boolean forceRefresh) {
+        if (forceRefresh) {
+            refreshRouteTable();
+        }
+        return this.regionRouteTable.findStartKeyOfNextRegion(key);
+    }
+
+    @Override
+    public RegionRouteTable getRegionRouteTable() {
+        return regionRouteTable;
+    }
+
+    @Override
+    public boolean transferLeader(final long regionId, final Peer peer, final boolean refreshConf) {
+        Requires.requireNonNull(peer, "peer");
+        Requires.requireNonNull(peer.getEndpoint(), "peer.endpoint");
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        final Configuration conf = RouteTable.getInstance().getConfiguration(raftGroupId);
+        final Status status = this.cliService.transferLeader(raftGroupId, conf, JRaftHelper.toJRaftPeerId(peer));
+        if (status.isOk()) {
+            if (refreshConf) {
+                refreshRouteConfiguration(regionId);
+            }
+            return true;
+        }
+        LOG.error("Fail to [transferLeader], [regionId: {}, peer: {}], status: {}.", regionId, peer, status);
+        return false;
+    }
+
+    @Override
+    public boolean addReplica(final long regionId, final Peer peer, final boolean refreshConf) {
+        Requires.requireNonNull(peer, "peer");
+        Requires.requireNonNull(peer.getEndpoint(), "peer.endpoint");
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        final Configuration conf = RouteTable.getInstance().getConfiguration(raftGroupId);
+        final Status status = this.cliService.addPeer(raftGroupId, conf, JRaftHelper.toJRaftPeerId(peer));
+        if (status.isOk()) {
+            if (refreshConf) {
+                refreshRouteConfiguration(regionId);
+            }
+            return true;
+        }
+        LOG.error("Fail to [addReplica], [regionId: {}, peer: {}], status: {}.", regionId, peer, status);
+        return false;
+    }
+
+    @Override
+    public boolean removeReplica(final long regionId, final Peer peer, final boolean refreshConf) {
+        Requires.requireNonNull(peer, "peer");
+        Requires.requireNonNull(peer.getEndpoint(), "peer.endpoint");
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        final Configuration conf = RouteTable.getInstance().getConfiguration(raftGroupId);
+        final Status status = this.cliService.removePeer(raftGroupId, conf, JRaftHelper.toJRaftPeerId(peer));
+        if (status.isOk()) {
+            if (refreshConf) {
+                refreshRouteConfiguration(regionId);
+            }
+            return true;
+        }
+        LOG.error("Fail to [removeReplica], [regionId: {}, peer: {}], status: {}.", regionId, peer, status);
+        return false;
+    }
+
+    @Override
+    public Endpoint getLeader(final long regionId, final boolean forceRefresh, final long timeoutMillis) {
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        PeerId leader = getLeader(raftGroupId, forceRefresh, timeoutMillis);
+        if (leader == null && !forceRefresh) {
+            // Could not found leader from cache, try again and force refresh cache
+            leader = getLeader(raftGroupId, true, timeoutMillis);
+        }
+        if (leader == null) {
+            throw new RouteTableException("no leader in group: " + raftGroupId);
+        }
+        return leader.getEndpoint();
+    }
+
+    protected PeerId getLeader(final String raftGroupId, final boolean forceRefresh, final long timeoutMillis) {
+        final RouteTable routeTable = RouteTable.getInstance();
+        if (forceRefresh) {
+            final long deadline = System.currentTimeMillis() + timeoutMillis;
+            final StringBuilder error = new StringBuilder();
+            // A newly launched raft group may not have been successful in the election,
+            // or in the 'leader-transfer' state, it needs to be re-tried
+            Throwable lastCause = null;
+            for (;;) {
+                try {
+                    final Status st = routeTable.refreshLeader(this.cliClientService, raftGroupId, 2000);
+                    if (st.isOk()) {
+                        break;
+                    }
+                    error.append(st.toString());
+                } catch (final InterruptedException e) {
+                    ThrowUtil.throwException(e);
+                } catch (final Throwable t) {
+                    lastCause = t;
+                    error.append(t.getMessage());
+                }
+                if (System.currentTimeMillis() < deadline) {
+                    LOG.debug("Fail to find leader, retry again, {}.", error);
+                    error.append(", ");
+                    try {
+                        Thread.sleep(10);
+                    } catch (final InterruptedException e) {
+                        ThrowUtil.throwException(e);
+                    }
+                } else {
+                    throw lastCause != null ? new RouteTableException(error.toString(), lastCause)
+                        : new RouteTableException(error.toString());
+                }
+            }
+
+            // we need refresh configuration for membership change
+            final long leftTime = deadline - System.currentTimeMillis();
+            if (leftTime > AT_LEAST_REQUIRED_MILLIS) {
+                try {
+                    RouteTable.getInstance().refreshConfiguration(this.cliClientService, raftGroupId, (int) leftTime);
+                } catch (final InterruptedException e) {
+                    ThrowUtil.throwException(e);
+                } catch (final TimeoutException ignored) {
+                }
+            }
+        }
+        return routeTable.selectLeader(raftGroupId);
+    }
+
+    @Override
+    public Endpoint getLuckyPeer(final long regionId, final boolean forceRefresh, final long timeoutMillis,
+                                 final Endpoint unExpect) {
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        final RouteTable routeTable = RouteTable.getInstance();
+        if (forceRefresh) {
+            final long deadline = System.currentTimeMillis() + timeoutMillis;
+            final StringBuilder error = new StringBuilder();
+            // A newly launched raft group may not have been successful in the election,
+            // or in the 'leader-transfer' state, it needs to be re-tried
+            for (;;) {
+                try {
+                    final Status st = routeTable.refreshConfiguration(this.cliClientService, raftGroupId, 5000);
+                    if (st.isOk()) {
+                        break;
+                    }
+                    error.append(st.toString());
+                } catch (final InterruptedException e) {
+                    ThrowUtil.throwException(e);
+                } catch (final TimeoutException e) {
+                    error.append(e.getMessage());
+                }
+                if (System.currentTimeMillis() < deadline) {
+                    LOG.debug("Fail to get peers, retry again, {}.", error);
+                    error.append(", ");
+                    try {
+                        Thread.sleep(5);
+                    } catch (final InterruptedException e) {
+                        ThrowUtil.throwException(e);
+                    }
+                } else {
+                    throw new RouteTableException(error.toString());
+                }
+            }
+        }
+        final Configuration configs = routeTable.getConfiguration(raftGroupId);
+        if (configs == null) {
+            throw new RouteTableException("empty configs in group: " + raftGroupId);
+        }
+        final List peerList = configs.getPeers();
+        if (peerList == null || peerList.isEmpty()) {
+            throw new RouteTableException("empty peers in group: " + raftGroupId);
+        }
+        final int size = peerList.size();
+        if (size == 1) {
+            return peerList.get(0).getEndpoint();
+        }
+        final RoundRobinLoadBalancer balancer = RoundRobinLoadBalancer.getInstance(regionId);
+        for (int i = 0; i < size; i++) {
+            final PeerId candidate = balancer.select(peerList);
+            final Endpoint luckyOne = candidate.getEndpoint();
+            if (!luckyOne.equals(unExpect)) {
+                return luckyOne;
+            }
+        }
+        throw new RouteTableException("have no choice in group(peers): " + raftGroupId);
+    }
+
+    @Override
+    public void refreshRouteConfiguration(final long regionId) {
+        final String raftGroupId = JRaftHelper.getJRaftGroupId(this.clusterName, regionId);
+        try {
+            getLeader(raftGroupId, true, 5000);
+            RouteTable.getInstance().refreshConfiguration(this.cliClientService, raftGroupId, 5000);
+        } catch (final Exception e) {
+            LOG.error("Fail to refresh route configuration for {}, {}.", regionId, StackTraceUtil.stackTrace(e));
+        }
+    }
+
+    @Override
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    @Override
+    public PlacementDriverRpcService getPdRpcService() {
+        return pdRpcService;
+    }
+
+    public RpcClient getRpcClient() {
+        return rpcClient;
+    }
+
+    protected void initRouteTableByRegion(final RegionRouteTableOptions opts) {
+        final long regionId = Requires.requireNonNull(opts.getRegionId(), "opts.regionId");
+        final byte[] startKey = opts.getStartKeyBytes();
+        final byte[] endKey = opts.getEndKeyBytes();
+        final String initialServerList = opts.getInitialServerList();
+        final Region region = new Region();
+        final Configuration conf = new Configuration();
+        // region
+        region.setId(regionId);
+        region.setStartKey(startKey);
+        region.setEndKey(endKey);
+        region.setRegionEpoch(new RegionEpoch(-1, -1));
+        // peers
+        Requires.requireTrue(Strings.isNotBlank(initialServerList), "opts.initialServerList is blank");
+        conf.parse(initialServerList);
+        region.setPeers(JRaftHelper.toPeerList(conf.listPeers()));
+        // update raft route table
+        RouteTable.getInstance().updateConfiguration(JRaftHelper.getJRaftGroupId(clusterName, regionId), conf);
+        this.regionRouteTable.addOrUpdateRegion(region);
+    }
+
+    protected Region getLocalRegionMetadata(final RegionEngineOptions opts) {
+        final long regionId = Requires.requireNonNull(opts.getRegionId(), "opts.regionId");
+        Requires.requireTrue(regionId >= Region.MIN_ID_WITH_MANUAL_CONF, "opts.regionId must >= "
+                                                                         + Region.MIN_ID_WITH_MANUAL_CONF);
+        Requires.requireTrue(regionId < Region.MAX_ID_WITH_MANUAL_CONF, "opts.regionId must < "
+                                                                        + Region.MAX_ID_WITH_MANUAL_CONF);
+        final byte[] startKey = opts.getStartKeyBytes();
+        final byte[] endKey = opts.getEndKeyBytes();
+        final String initialServerList = opts.getInitialServerList();
+        final Region region = new Region();
+        final Configuration conf = new Configuration();
+        // region
+        region.setId(regionId);
+        region.setStartKey(startKey);
+        region.setEndKey(endKey);
+        region.setRegionEpoch(new RegionEpoch(-1, -1));
+        // peers
+        Requires.requireTrue(Strings.isNotBlank(initialServerList), "opts.initialServerList is blank");
+        conf.parse(initialServerList);
+        region.setPeers(JRaftHelper.toPeerList(conf.listPeers()));
+        this.regionRouteTable.addOrUpdateRegion(region);
+        return region;
+    }
+
+    protected void initCli(CliOptions cliOpts) {
+        if (cliOpts == null) {
+            cliOpts = new CliOptions();
+            cliOpts.setTimeoutMs(5000);
+            cliOpts.setMaxRetry(3);
+        }
+        this.cliService = RaftServiceFactory.createAndInitCliService(cliOpts);
+        this.cliClientService = ((CliServiceImpl) this.cliService).getCliClientService();
+        Requires.requireNonNull(this.cliClientService, "cliClientService");
+        this.rpcClient = ((AbstractClientService) this.cliClientService).getRpcClient();
+    }
+
+    protected abstract void refreshRouteTable();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/DefaultPlacementDriverRpcService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/DefaultPlacementDriverRpcService.java
new file mode 100644
index 0000000..98f6f40
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/DefaultPlacementDriverRpcService.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseResponse;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.ErrorsHelper;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.rpc.ExtSerializerSupports;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.InvokeContext;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ * @author jiachun.fjc
+ */
+public class DefaultPlacementDriverRpcService implements PlacementDriverRpcService {
+
+    private static final Logger         LOG = LoggerFactory.getLogger(DefaultPlacementDriverRpcService.class);
+
+    private final PlacementDriverClient pdClient;
+    private final RpcClient             rpcClient;
+
+    private ThreadPoolExecutor          rpcCallbackExecutor;
+    private int                         rpcTimeoutMillis;
+
+    private boolean                     started;
+
+    public DefaultPlacementDriverRpcService(PlacementDriverClient pdClient) {
+        this.pdClient = pdClient;
+        this.rpcClient = ((AbstractPlacementDriverClient) pdClient).getRpcClient();
+    }
+
+    @Override
+    public synchronized boolean init(final RpcOptions opts) {
+        if (this.started) {
+            LOG.info("[DefaultPlacementDriverRpcService] already started.");
+            return true;
+        }
+        this.rpcCallbackExecutor = createRpcCallbackExecutor(opts);
+        this.rpcTimeoutMillis = opts.getRpcTimeoutMillis();
+        Requires.requireTrue(this.rpcTimeoutMillis > 0, "opts.rpcTimeoutMillis must > 0");
+        LOG.info("[DefaultPlacementDriverRpcService] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.rpcCallbackExecutor);
+        this.started = false;
+        LOG.info("[DefaultPlacementDriverRpcService] shutdown successfully.");
+    }
+
+    @Override
+    public  CompletableFuture callPdServerWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                                        final Errors lastCause) {
+        final boolean forceRefresh = ErrorsHelper.isInvalidPeer(lastCause);
+        final Endpoint endpoint = this.pdClient.getPdLeader(forceRefresh, this.rpcTimeoutMillis);
+        internalCallPdWithRpc(endpoint, request, closure);
+        return closure.future();
+    }
+
+    private  void internalCallPdWithRpc(final Endpoint endpoint, final BaseRequest request,
+                                           final FailoverClosure closure) {
+        final InvokeContext invokeCtx = new InvokeContext();
+        invokeCtx.put(BoltRpcClient.BOLT_CTX, ExtSerializerSupports.getInvokeContext());
+        final InvokeCallback invokeCallback = new InvokeCallback() {
+
+            @Override
+            public void complete(final Object result, final Throwable err) {
+                if (err == null) {
+                    final BaseResponse response = (BaseResponse) result;
+                    if (response.isSuccess()) {
+                        closure.setData(response.getValue());
+                        closure.run(Status.OK());
+                    } else {
+                        closure.setError(response.getError());
+                        closure.run(new Status(-1, "RPC failed with address: %s, response: %s", endpoint, response));
+                    }
+                } else {
+                    closure.failure(err);
+                }
+            }
+
+            @Override
+            public Executor executor() {
+                return rpcCallbackExecutor;
+            }
+        };
+
+        try {
+            this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback, this.rpcTimeoutMillis);
+        } catch (final Throwable t) {
+            closure.failure(t);
+        }
+    }
+
+    private ThreadPoolExecutor createRpcCallbackExecutor(final RpcOptions opts) {
+        final int callbackExecutorCorePoolSize = opts.getCallbackExecutorCorePoolSize();
+        final int callbackExecutorMaximumPoolSize = opts.getCallbackExecutorMaximumPoolSize();
+        if (callbackExecutorCorePoolSize <= 0 || callbackExecutorMaximumPoolSize <= 0) {
+            return null;
+        }
+
+        final String name = "rheakv-pd-rpc-callback";
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(true) //
+            .coreThreads(callbackExecutorCorePoolSize) //
+            .maximumThreads(callbackExecutorMaximumPoolSize) //
+            .keepAliveSeconds(120L) //
+            .workQueue(new ArrayBlockingQueue<>(opts.getCallbackExecutorQueueCapacity())) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(new CallerRunsPolicyWithReport(name)) //
+            .build();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/FakePlacementDriverClient.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/FakePlacementDriverClient.java
new file mode 100644
index 0000000..23c2501
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/FakePlacementDriverClient.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Single raft group, no need for a real PD role.
+ *
+ * @author jiachun.fjc
+ */
+public class FakePlacementDriverClient extends AbstractPlacementDriverClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(FakePlacementDriverClient.class);
+
+    private boolean             started;
+
+    public FakePlacementDriverClient(long clusterId, String clusterName) {
+        super(clusterId, clusterName);
+    }
+
+    @Override
+    public synchronized boolean init(final PlacementDriverOptions opts) {
+        if (this.started) {
+            LOG.info("[FakePlacementDriverClient] already started.");
+            return true;
+        }
+        super.init(opts);
+        LOG.info("[FakePlacementDriverClient] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        super.shutdown();
+        LOG.info("[FakePlacementDriverClient] shutdown successfully.");
+    }
+
+    @Override
+    protected void refreshRouteTable() {
+        // NO-OP
+    }
+
+    @Override
+    public Store getStoreMetadata(final StoreEngineOptions opts) {
+        final Store store = new Store();
+        final List rOptsList = opts.getRegionEngineOptionsList();
+        final List regionList = Lists.newArrayListWithCapacity(rOptsList.size());
+        store.setId(-1);
+        store.setEndpoint(opts.getServerAddress());
+        for (final RegionEngineOptions rOpts : rOptsList) {
+            regionList.add(getLocalRegionMetadata(rOpts));
+        }
+        store.setRegions(regionList);
+        return store;
+    }
+
+    @Override
+    public Endpoint getPdLeader(final boolean forceRefresh, final long timeoutMillis) {
+        throw new UnsupportedOperationException();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/HeartbeatSender.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/HeartbeatSender.java
new file mode 100644
index 0000000..b2e607f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/HeartbeatSender.java
@@ -0,0 +1,307 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.errors.ErrorsHelper;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.metadata.TimeInterval;
+import com.alipay.sofa.jraft.rhea.options.HeartbeatOptions;
+import com.alipay.sofa.jraft.rhea.rpc.ExtSerializerSupports;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DiscardOldPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rpc.InvokeCallback;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.BoltRpcClient;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+import com.alipay.sofa.jraft.util.timer.HashedWheelTimer;
+import com.alipay.sofa.jraft.util.timer.Timeout;
+import com.alipay.sofa.jraft.util.timer.TimerTask;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class HeartbeatSender implements Lifecycle {
+
+    private static final Logger         LOG = LoggerFactory.getLogger(HeartbeatSender.class);
+
+    private final StoreEngine           storeEngine;
+    private final PlacementDriverClient pdClient;
+    private final RpcClient             rpcClient;
+
+    private StatsCollector              statsCollector;
+    private InstructionProcessor        instructionProcessor;
+    private int                         heartbeatRpcTimeoutMillis;
+    private ThreadPoolExecutor          heartbeatRpcCallbackExecutor;
+    private HashedWheelTimer            heartbeatTimer;
+
+    private boolean                     started;
+
+    public HeartbeatSender(StoreEngine storeEngine) {
+        this.storeEngine = storeEngine;
+        this.pdClient = storeEngine.getPlacementDriverClient();
+        this.rpcClient = ((AbstractPlacementDriverClient) this.pdClient).getRpcClient();
+    }
+
+    @Override
+    public synchronized boolean init(final HeartbeatOptions opts) {
+        if (this.started) {
+            LOG.info("[HeartbeatSender] already started.");
+            return true;
+        }
+        this.statsCollector = new StatsCollector(this.storeEngine);
+        this.instructionProcessor = new InstructionProcessor(this.storeEngine);
+        this.heartbeatTimer = new HashedWheelTimer(new NamedThreadFactory("heartbeat-timer", true), 50,
+            TimeUnit.MILLISECONDS, 4096);
+        this.heartbeatRpcTimeoutMillis = opts.getHeartbeatRpcTimeoutMillis();
+        if (this.heartbeatRpcTimeoutMillis <= 0) {
+            throw new IllegalArgumentException("Heartbeat rpc timeout millis must > 0, "
+                                               + this.heartbeatRpcTimeoutMillis);
+        }
+        final String name = "rheakv-heartbeat-callback";
+        this.heartbeatRpcCallbackExecutor = ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(true) //
+            .coreThreads(4) //
+            .maximumThreads(4) //
+            .keepAliveSeconds(120L) //
+            .workQueue(new ArrayBlockingQueue<>(1024)) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(new DiscardOldPolicyWithReport(name)) //
+            .build();
+        final long storeHeartbeatIntervalSeconds = opts.getStoreHeartbeatIntervalSeconds();
+        final long regionHeartbeatIntervalSeconds = opts.getRegionHeartbeatIntervalSeconds();
+        if (storeHeartbeatIntervalSeconds <= 0) {
+            throw new IllegalArgumentException("Store heartbeat interval seconds must > 0, "
+                                               + storeHeartbeatIntervalSeconds);
+        }
+        if (regionHeartbeatIntervalSeconds <= 0) {
+            throw new IllegalArgumentException("Region heartbeat interval seconds must > 0, "
+                                               + regionHeartbeatIntervalSeconds);
+        }
+        final long now = System.currentTimeMillis();
+        final StoreHeartbeatTask storeHeartbeatTask = new StoreHeartbeatTask(storeHeartbeatIntervalSeconds, now, false);
+        final RegionHeartbeatTask regionHeartbeatTask = new RegionHeartbeatTask(regionHeartbeatIntervalSeconds, now,
+            false);
+        this.heartbeatTimer.newTimeout(storeHeartbeatTask, storeHeartbeatTask.getNextDelay(), TimeUnit.SECONDS);
+        this.heartbeatTimer.newTimeout(regionHeartbeatTask, regionHeartbeatTask.getNextDelay(), TimeUnit.SECONDS);
+        LOG.info("[HeartbeatSender] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.heartbeatRpcCallbackExecutor);
+        if (this.heartbeatTimer != null) {
+            this.heartbeatTimer.stop();
+        }
+    }
+
+    private void sendStoreHeartbeat(final long nextDelay, final boolean forceRefreshLeader, final long lastTime) {
+        final long now = System.currentTimeMillis();
+        final StoreHeartbeatRequest request = new StoreHeartbeatRequest();
+        request.setClusterId(this.storeEngine.getClusterId());
+        final TimeInterval timeInterval = new TimeInterval(lastTime, now);
+        final StoreStats stats = this.statsCollector.collectStoreStats(timeInterval);
+        request.setStats(stats);
+        final HeartbeatClosure closure = new HeartbeatClosure() {
+
+            @Override
+            public void run(final Status status) {
+                final boolean forceRefresh = !status.isOk() && ErrorsHelper.isInvalidPeer(getError());
+                final StoreHeartbeatTask nexTask = new StoreHeartbeatTask(nextDelay, now, forceRefresh);
+                heartbeatTimer.newTimeout(nexTask, nexTask.getNextDelay(), TimeUnit.SECONDS);
+            }
+        };
+        final Endpoint endpoint = this.pdClient.getPdLeader(forceRefreshLeader, this.heartbeatRpcTimeoutMillis);
+        callAsyncWithRpc(endpoint, request, closure);
+    }
+
+    private void sendRegionHeartbeat(final long nextDelay, final long lastTime, final boolean forceRefreshLeader) {
+        final long now = System.currentTimeMillis();
+        final RegionHeartbeatRequest request = new RegionHeartbeatRequest();
+        request.setClusterId(this.storeEngine.getClusterId());
+        request.setStoreId(this.storeEngine.getStoreId());
+        request.setLeastKeysOnSplit(this.storeEngine.getStoreOpts().getLeastKeysOnSplit());
+        final List regionIdList = this.storeEngine.getLeaderRegionIds();
+        if (regionIdList.isEmpty()) {
+            // So sad, there is no even a region leader :(
+            final RegionHeartbeatTask nextTask = new RegionHeartbeatTask(nextDelay, now, false);
+            this.heartbeatTimer.newTimeout(nextTask, nextTask.getNextDelay(), TimeUnit.SECONDS);
+            if (LOG.isInfoEnabled()) {
+                LOG.info("So sad, there is no even a region leader on [clusterId:{}, storeId: {}, endpoint:{}].",
+                    this.storeEngine.getClusterId(), this.storeEngine.getStoreId(), this.storeEngine.getSelfEndpoint());
+            }
+            return;
+        }
+        final List> regionStatsList = Lists.newArrayListWithCapacity(regionIdList.size());
+        final TimeInterval timeInterval = new TimeInterval(lastTime, now);
+        for (final Long regionId : regionIdList) {
+            final Region region = this.pdClient.getRegionById(regionId);
+            final RegionStats stats = this.statsCollector.collectRegionStats(region, timeInterval);
+            if (stats == null) {
+                continue;
+            }
+            regionStatsList.add(Pair.of(region, stats));
+        }
+        request.setRegionStatsList(regionStatsList);
+        final HeartbeatClosure> closure = new HeartbeatClosure>() {
+
+            @Override
+            public void run(final Status status) {
+                final boolean isOk = status.isOk();
+                if (isOk) {
+                    final List instructions = getResult();
+                    if (instructions != null && !instructions.isEmpty()) {
+                        instructionProcessor.process(instructions);
+                    }
+                }
+                final boolean forceRefresh = !isOk && ErrorsHelper.isInvalidPeer(getError());
+                final RegionHeartbeatTask nextTask = new RegionHeartbeatTask(nextDelay, now, forceRefresh);
+                heartbeatTimer.newTimeout(nextTask, nextTask.getNextDelay(), TimeUnit.SECONDS);
+            }
+        };
+        final Endpoint endpoint = this.pdClient.getPdLeader(forceRefreshLeader, this.heartbeatRpcTimeoutMillis);
+        callAsyncWithRpc(endpoint, request, closure);
+    }
+
+    private  void callAsyncWithRpc(final Endpoint endpoint, final BaseRequest request,
+                                      final HeartbeatClosure closure) {
+        final com.alipay.sofa.jraft.rpc.InvokeContext invokeCtx = new com.alipay.sofa.jraft.rpc.InvokeContext();
+        invokeCtx.put(BoltRpcClient.BOLT_CTX, ExtSerializerSupports.getInvokeContext());
+        final InvokeCallback invokeCallback = new InvokeCallback() {
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public void complete(final Object result, final Throwable err) {
+                if (err == null) {
+                    final BaseResponse response = (BaseResponse) result;
+                    if (response.isSuccess()) {
+                        closure.setResult((V) response.getValue());
+                        closure.run(Status.OK());
+                    } else {
+                        closure.setError(response.getError());
+                        closure.run(new Status(-1, "RPC failed with address: %s, response: %s", endpoint, response));
+                    }
+                } else {
+                    closure.run(new Status(-1, err.getMessage()));
+                }
+            }
+
+            @Override
+            public Executor executor() {
+                return heartbeatRpcCallbackExecutor;
+            }
+        };
+
+        try {
+            this.rpcClient.invokeAsync(endpoint, request, invokeCtx, invokeCallback, this.heartbeatRpcTimeoutMillis);
+        } catch (final Throwable t) {
+            closure.run(new Status(-1, t.getMessage()));
+        }
+    }
+
+    private static abstract class HeartbeatClosure extends BaseKVStoreClosure {
+
+        private volatile V result;
+
+        public V getResult() {
+            return result;
+        }
+
+        public void setResult(V result) {
+            this.result = result;
+        }
+    }
+
+    private final class StoreHeartbeatTask implements TimerTask {
+
+        private final long    nextDelay;
+        private final long    lastTime;
+        private final boolean forceRefreshLeader;
+
+        private StoreHeartbeatTask(long nextDelay, long lastTime, boolean forceRefreshLeader) {
+            this.nextDelay = nextDelay;
+            this.lastTime = lastTime;
+            this.forceRefreshLeader = forceRefreshLeader;
+        }
+
+        @Override
+        public void run(final Timeout timeout) throws Exception {
+            try {
+                sendStoreHeartbeat(this.nextDelay, this.forceRefreshLeader, this.lastTime);
+            } catch (final Throwable t) {
+                LOG.error("Caught a error on sending [StoreHeartbeat]: {}.", StackTraceUtil.stackTrace(t));
+            }
+        }
+
+        public long getNextDelay() {
+            return nextDelay;
+        }
+    }
+
+    private final class RegionHeartbeatTask implements TimerTask {
+
+        private final long    nextDelay;
+        private final long    lastTime;
+        private final boolean forceRefreshLeader;
+
+        private RegionHeartbeatTask(long nextDelay, long lastTime, boolean forceRefreshLeader) {
+            this.nextDelay = nextDelay;
+            this.lastTime = lastTime;
+            this.forceRefreshLeader = forceRefreshLeader;
+        }
+
+        @Override
+        public void run(final Timeout timeout) throws Exception {
+            try {
+                sendRegionHeartbeat(this.nextDelay, this.lastTime, this.forceRefreshLeader);
+            } catch (final Throwable t) {
+                LOG.error("Caught a error on sending [RegionHeartbeat]: {}.", StackTraceUtil.stackTrace(t));
+            }
+        }
+
+        public long getNextDelay() {
+            return nextDelay;
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/InstructionProcessor.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/InstructionProcessor.java
new file mode 100644
index 0000000..ba7322c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/InstructionProcessor.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.RegionEngine;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Processing the instructions from the placement driver server.
+ *
+ * @author jiachun.fjc
+ */
+public class InstructionProcessor {
+
+    private static final Logger LOG = LoggerFactory.getLogger(InstructionProcessor.class);
+
+    private final StoreEngine   storeEngine;
+
+    public InstructionProcessor(StoreEngine storeEngine) {
+        this.storeEngine = storeEngine;
+    }
+
+    public void process(final List instructions) {
+        LOG.info("Received instructions: {}.", instructions);
+        for (final Instruction instruction : instructions) {
+            if (!checkInstruction(instruction)) {
+                continue;
+            }
+            processSplit(instruction);
+            processTransferLeader(instruction);
+        }
+    }
+
+    private boolean processSplit(final Instruction instruction) {
+        try {
+            final Instruction.RangeSplit rangeSplit = instruction.getRangeSplit();
+            if (rangeSplit == null) {
+                return false;
+            }
+            final Long newRegionId = rangeSplit.getNewRegionId();
+            if (newRegionId == null) {
+                LOG.error("RangeSplit#newRegionId must not be null, {}.", instruction);
+                return false;
+            }
+            final Region region = instruction.getRegion();
+            final long regionId = region.getId();
+            final RegionEngine engine = this.storeEngine.getRegionEngine(regionId);
+            if (engine == null) {
+                LOG.error("Could not found regionEngine, {}.", instruction);
+                return false;
+            }
+            if (!region.equals(engine.getRegion())) {
+                LOG.warn("Instruction [{}] is out of date.", instruction);
+                return false;
+            }
+            final CompletableFuture future = new CompletableFuture<>();
+            this.storeEngine.applySplit(regionId, newRegionId, new BaseKVStoreClosure() {
+
+                @Override
+                public void run(Status status) {
+                    future.complete(status);
+                }
+            });
+            final Status status = future.get(20, TimeUnit.SECONDS);
+            final boolean ret = status.isOk();
+            if (ret) {
+                LOG.info("Range-split succeeded, instruction: {}.", instruction);
+            } else {
+                LOG.warn("Range-split failed: {}, instruction: {}.", status, instruction);
+            }
+            return ret;
+        } catch (final Throwable t) {
+            LOG.error("Caught an exception on #processSplit: {}.", StackTraceUtil.stackTrace(t));
+            return false;
+        }
+    }
+
+    private boolean processTransferLeader(final Instruction instruction) {
+        try {
+            final Instruction.TransferLeader transferLeader = instruction.getTransferLeader();
+            if (transferLeader == null) {
+                return false;
+            }
+            final Endpoint toEndpoint = transferLeader.getMoveToEndpoint();
+            if (toEndpoint == null) {
+                LOG.error("TransferLeader#toEndpoint must not be null, {}.", instruction);
+                return false;
+            }
+            final Region region = instruction.getRegion();
+            final long regionId = region.getId();
+            final RegionEngine engine = this.storeEngine.getRegionEngine(regionId);
+            if (engine == null) {
+                LOG.error("Could not found regionEngine, {}.", instruction);
+                return false;
+            }
+            if (!region.equals(engine.getRegion())) {
+                LOG.warn("Instruction [{}] is out of date.", instruction);
+                return false;
+            }
+            return engine.transferLeadershipTo(toEndpoint);
+        } catch (final Throwable t) {
+            LOG.error("Caught an exception on #processTransferLeader: {}.", StackTraceUtil.stackTrace(t));
+            return false;
+        }
+    }
+
+    private boolean checkInstruction(final Instruction instruction) {
+        if (instruction == null) {
+            LOG.warn("Null instructions element.");
+            return false;
+        }
+        if (instruction.getRegion() == null) {
+            LOG.warn("Null region with instruction: {}.", instruction);
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/MetadataRpcClient.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/MetadataRpcClient.java
new file mode 100644
index 0000000..1474b20
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/MetadataRpcClient.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.rhea.client.FutureHelper;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryRunner;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.FailoverClosureImpl;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MetadataRpcClient {
+
+    private final PlacementDriverRpcService pdRpcService;
+    private final int                       failoverRetries;
+
+    public MetadataRpcClient(PlacementDriverRpcService pdRpcService, int failoverRetries) {
+        this.pdRpcService = pdRpcService;
+        this.failoverRetries = failoverRetries;
+    }
+
+    /**
+     * Returns the specified cluster information.
+     */
+    public Cluster getClusterInfo(final long clusterId) {
+        final CompletableFuture future = new CompletableFuture<>();
+        internalGetClusterInfo(clusterId, future, this.failoverRetries, null);
+        return FutureHelper.get(future);
+    }
+
+    private void internalGetClusterInfo(final long clusterId, final CompletableFuture future,
+                                        final int retriesLeft, final Errors lastCause) {
+        final RetryRunner retryRunner = retryCause -> internalGetClusterInfo(clusterId, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        final GetClusterInfoRequest request = new GetClusterInfoRequest();
+        request.setClusterId(clusterId);
+        this.pdRpcService.callPdServerWithRpc(request, closure, lastCause);
+    }
+
+    /**
+     * The pd server stores the storeIds of all nodes.
+     * This method provides a lookup for the storeId according
+     * to the host.  If there is no value, then a globally
+     * unique storeId is created.
+     */
+    public Long getOrCreateStoreId(final long clusterId, final Endpoint endpoint) {
+        final CompletableFuture future = new CompletableFuture<>();
+        internalGetOrCreateStoreId(clusterId, endpoint, future, this.failoverRetries, null);
+        return FutureHelper.get(future);
+    }
+
+    private void internalGetOrCreateStoreId(final long clusterId, final Endpoint endpoint,
+                                            final CompletableFuture future, final int retriesLeft,
+                                            final Errors lastCause) {
+        final RetryRunner retryRunner = retryCause -> internalGetOrCreateStoreId(clusterId, endpoint, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        final GetStoreIdRequest request = new GetStoreIdRequest();
+        request.setClusterId(clusterId);
+        request.setEndpoint(endpoint);
+        this.pdRpcService.callPdServerWithRpc(request, closure, lastCause);
+    }
+
+    /**
+     * Query the store information by the host.  If the result
+     * is a empty instance, the caller needs to use its own local
+     * configuration.
+     */
+    public Store getStoreInfo(final long clusterId, final Endpoint selfEndpoint) {
+        final CompletableFuture future = new CompletableFuture<>();
+        internalGetStoreInfo(clusterId, selfEndpoint, future, this.failoverRetries, null);
+        return FutureHelper.get(future);
+    }
+
+    private void internalGetStoreInfo(final long clusterId, final Endpoint selfEndpoint,
+                                      final CompletableFuture future, final int retriesLeft,
+                                      final Errors lastCause) {
+        final RetryRunner retryRunner = retryCause -> internalGetStoreInfo(clusterId, selfEndpoint, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        final GetStoreInfoRequest request = new GetStoreInfoRequest();
+        request.setClusterId(clusterId);
+        request.setEndpoint(selfEndpoint);
+        this.pdRpcService.callPdServerWithRpc(request, closure, lastCause);
+    }
+
+    /**
+     * Update the store information by the storeId.
+     */
+    public Store updateStoreInfo(final long clusterId, final Store store) {
+        final CompletableFuture future = new CompletableFuture<>();
+        internalUpdateStoreInfo(clusterId, store, future, 1, null);
+        return FutureHelper.get(future);
+    }
+
+    private void internalUpdateStoreInfo(final long clusterId, final Store store, final CompletableFuture future,
+                                         final int retriesLeft, final Errors lastCause) {
+        final RetryRunner retryRunner = retryCause -> internalUpdateStoreInfo(clusterId, store, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        final SetStoreInfoRequest request = new SetStoreInfoRequest();
+        request.setClusterId(clusterId);
+        request.setStore(store);
+        this.pdRpcService.callPdServerWithRpc(request, closure, lastCause);
+    }
+
+    /**
+     * Create a globally unique regionId.
+     */
+    public Long createRegionId(final long clusterId, final Endpoint endpoint) {
+        final CompletableFuture future = new CompletableFuture<>();
+        internalCreateRegionId(clusterId, endpoint, future, this.failoverRetries, null);
+        return FutureHelper.get(future);
+    }
+
+    private void internalCreateRegionId(final long clusterId, final Endpoint endpoint,
+                                        final CompletableFuture future, final int retriesLeft,
+                                        final Errors lastCause) {
+        final RetryRunner retryRunner = retryCause -> internalCreateRegionId(clusterId, endpoint, future,
+                retriesLeft - 1, retryCause);
+        final FailoverClosure closure = new FailoverClosureImpl<>(future, retriesLeft, retryRunner);
+        final CreateRegionIdRequest request = new CreateRegionIdRequest();
+        request.setClusterId(clusterId);
+        request.setEndpoint(endpoint);
+        this.pdRpcService.callPdServerWithRpc(request, closure, lastCause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverClient.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverClient.java
new file mode 100644
index 0000000..e7293e9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverClient.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.rhea.client.RegionRouteTable;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Placement driver client
+ *
+ * @author jiachun.fjc
+ */
+public interface PlacementDriverClient extends Lifecycle {
+
+    /**
+     * Returns the cluster id.
+     */
+    long getClusterId();
+
+    /**
+     * Query the region by region id.
+     */
+    Region getRegionById(final long regionId);
+
+    /**
+     * Returns the region to which the key belongs.
+     */
+    Region findRegionByKey(final byte[] key, final boolean forceRefresh);
+
+    /**
+     * Returns the regions to which the keys belongs.
+     */
+    Map> findRegionsByKeys(final List keys, final boolean forceRefresh);
+
+    /**
+     * Returns the regions to which the keys belongs.
+     */
+    Map> findRegionsByKvEntries(final List kvEntries, final boolean forceRefresh);
+
+    /**
+     * Returns the regions to which the keys belongs.
+     */
+    Map> findRegionsByCASEntries(final List casEntries, final boolean forceRefresh);
+
+    /**
+     * Returns the list of regions covered by startKey and endKey.
+     */
+    List findRegionsByKeyRange(final byte[] startKey, final byte[] endKey, final boolean forceRefresh);
+
+    /**
+     * Returns the startKey of next region.
+     */
+    byte[] findStartKeyOfNextRegion(final byte[] key, final boolean forceRefresh);
+
+    /**
+     * Returns the regionRouteTable instance.
+     */
+    RegionRouteTable getRegionRouteTable();
+
+    /**
+     * Returns the store metadata of the current instance's store.
+     * Construct initial data based on the configuration file if
+     * the data on {@link PlacementDriverClient} is empty.
+     */
+    Store getStoreMetadata(final StoreEngineOptions opts);
+
+    /**
+     * Get the specified region leader communication address.
+     */
+    Endpoint getLeader(final long regionId, final boolean forceRefresh, final long timeoutMillis);
+
+    /**
+     * Get the specified region random peer communication address,
+     * format: [ip:port]
+     */
+    Endpoint getLuckyPeer(final long regionId, final boolean forceRefresh, final long timeoutMillis,
+                          final Endpoint unExpect);
+
+    /**
+     * Refresh the routing information of the specified region
+     */
+    void refreshRouteConfiguration(final long regionId);
+
+    /**
+     * Transfer leader to specified peer.
+     */
+    boolean transferLeader(final long regionId, final Peer peer, final boolean refreshConf);
+
+    /**
+     * Join the specified region group.
+     */
+    boolean addReplica(final long regionId, final Peer peer, final boolean refreshConf);
+
+    /**
+     * Depart from the specified region group.
+     */
+    boolean removeReplica(final long regionId, final Peer peer, final boolean refreshConf);
+
+    /**
+     * Returns raft cluster prefix id.
+     */
+    String getClusterName();
+
+    /**
+     * Get the placement driver server's leader communication address.
+     */
+    Endpoint getPdLeader(final boolean forceRefresh, final long timeoutMillis);
+
+    /**
+     * Returns the pd rpc service client.
+     */
+    PlacementDriverRpcService getPdRpcService();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverRpcService.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverRpcService.java
new file mode 100644
index 0000000..9eb2ae4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/PlacementDriverRpcService.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+
+/**
+ * Placement driver's rpc client for sending requests and receiving responses.
+ *
+ * @author jiachun.fjc
+ */
+public interface PlacementDriverRpcService extends Lifecycle {
+
+    /**
+     * Send requests to the remote placement driver nodes.
+     *
+     * @param request   request data
+     * @param closure   callback for failover strategy
+     * @param lastCause the exception information held by the last call
+     *                  failed, the initial value is null
+     * @param        the type of response
+     * @return a future with response
+     */
+     CompletableFuture callPdServerWithRpc(final BaseRequest request, final FailoverClosure closure,
+                                                 final Errors lastCause);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/RemotePlacementDriverClient.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/RemotePlacementDriverClient.java
new file mode 100644
index 0000000..cd4562a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/RemotePlacementDriverClient.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.errors.RouteTableException;
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RemotePlacementDriverClient extends AbstractPlacementDriverClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RemotePlacementDriverClient.class);
+
+    private String              pdGroupId;
+    private MetadataRpcClient   metadataRpcClient;
+
+    private boolean             started;
+
+    public RemotePlacementDriverClient(long clusterId, String clusterName) {
+        super(clusterId, clusterName);
+    }
+
+    @Override
+    public synchronized boolean init(final PlacementDriverOptions opts) {
+        if (this.started) {
+            LOG.info("[RemotePlacementDriverClient] already started.");
+            return true;
+        }
+        super.init(opts);
+        this.pdGroupId = opts.getPdGroupId();
+        if (Strings.isBlank(this.pdGroupId)) {
+            throw new IllegalArgumentException("opts.pdGroup id must not be blank");
+        }
+        final String initialPdServers = opts.getInitialPdServerList();
+        if (Strings.isBlank(initialPdServers)) {
+            throw new IllegalArgumentException("opts.initialPdServerList must not be blank");
+        }
+        RouteTable.getInstance().updateConfiguration(this.pdGroupId, initialPdServers);
+        this.metadataRpcClient = new MetadataRpcClient(super.pdRpcService, 3);
+        refreshRouteTable();
+        LOG.info("[RemotePlacementDriverClient] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        super.shutdown();
+        LOG.info("[RemotePlacementDriverClient] shutdown successfully.");
+    }
+
+    @Override
+    protected void refreshRouteTable() {
+        final Cluster cluster = this.metadataRpcClient.getClusterInfo(this.clusterId);
+        if (cluster == null) {
+            LOG.warn("Cluster info is empty: {}.", this.clusterId);
+            return;
+        }
+        final List stores = cluster.getStores();
+        if (stores == null || stores.isEmpty()) {
+            LOG.error("Stores info is empty: {}.", this.clusterId);
+            return;
+        }
+        for (final Store store : stores) {
+            final List regions = store.getRegions();
+            if (regions == null || regions.isEmpty()) {
+                LOG.error("Regions info is empty: {} - {}.", this.clusterId, store.getId());
+                continue;
+            }
+            for (final Region region : regions) {
+                super.regionRouteTable.addOrUpdateRegion(region);
+            }
+        }
+    }
+
+    @Override
+    public Store getStoreMetadata(final StoreEngineOptions opts) {
+        final Endpoint selfEndpoint = opts.getServerAddress();
+        // remote conf is the preferred
+        final Store remoteStore = this.metadataRpcClient.getStoreInfo(this.clusterId, selfEndpoint);
+        if (!remoteStore.isEmpty()) {
+            final List regions = remoteStore.getRegions();
+            for (final Region region : regions) {
+                super.regionRouteTable.addOrUpdateRegion(region);
+            }
+            return remoteStore;
+        }
+        // local conf
+        final Store localStore = new Store();
+        final List rOptsList = opts.getRegionEngineOptionsList();
+        final List regionList = Lists.newArrayListWithCapacity(rOptsList.size());
+        localStore.setId(remoteStore.getId());
+        localStore.setEndpoint(selfEndpoint);
+        for (final RegionEngineOptions rOpts : rOptsList) {
+            regionList.add(getLocalRegionMetadata(rOpts));
+        }
+        localStore.setRegions(regionList);
+        this.metadataRpcClient.updateStoreInfo(this.clusterId, localStore);
+        return localStore;
+    }
+
+    @Override
+    public Endpoint getPdLeader(final boolean forceRefresh, final long timeoutMillis) {
+        PeerId leader = getLeader(this.pdGroupId, forceRefresh, timeoutMillis);
+        if (leader == null && !forceRefresh) {
+            leader = getLeader(this.pdGroupId, true, timeoutMillis);
+        }
+        if (leader == null) {
+            throw new RouteTableException("no placement driver leader in group: " + this.pdGroupId);
+        }
+        return new Endpoint(leader.getIp(), leader.getPort());
+    }
+
+    public MetadataRpcClient getMetadataRpcClient() {
+        return metadataRpcClient;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/StatsCollector.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/StatsCollector.java
new file mode 100644
index 0000000..4fa33d8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/client/pd/StatsCollector.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client.pd;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.metadata.TimeInterval;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetricNames;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.rocks.support.RocksStatistics;
+import com.alipay.sofa.jraft.rhea.storage.BaseRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.codahale.metrics.Counter;
+
+import static org.rocksdb.TickerType.BYTES_READ;
+import static org.rocksdb.TickerType.BYTES_WRITTEN;
+import static org.rocksdb.TickerType.NUMBER_KEYS_READ;
+import static org.rocksdb.TickerType.NUMBER_KEYS_WRITTEN;
+import static org.rocksdb.TickerType.NUMBER_MULTIGET_BYTES_READ;
+import static org.rocksdb.TickerType.NUMBER_MULTIGET_KEYS_READ;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StatsCollector {
+
+    private static final Logger     LOG = LoggerFactory.getLogger(StatsCollector.class);
+
+    private final StoreEngine       storeEngine;
+    private final BaseRawKVStore rawKVStore;
+    private final RocksRawKVStore   rocksRawKVStore;
+
+    public StatsCollector(StoreEngine storeEngine) {
+        this.storeEngine = storeEngine;
+        this.rawKVStore = storeEngine.getRawKVStore();
+        RocksRawKVStore store = null;
+        if (storeEngine.getStoreOpts().getStorageType() == StorageType.RocksDB) {
+            store = (RocksRawKVStore) rawKVStore;
+        }
+        this.rocksRawKVStore = store;
+    }
+
+    public StoreStats collectStoreStats(final TimeInterval timeInterval) {
+        final StoreStats stats = new StoreStats();
+        stats.setStoreId(this.storeEngine.getStoreId());
+        // Capacity for the store
+        stats.setCapacity(this.storeEngine.getTotalSpace());
+        // Available size for the store
+        stats.setAvailable(this.storeEngine.getUsableSpace());
+        // Total region count in this store
+        stats.setRegionCount(this.storeEngine.getRegionCount());
+        // Leader region count in this store
+        stats.setLeaderRegionCount(this.storeEngine.getLeaderRegionCount());
+        // Current sending snapshot count
+        // TODO
+        // Current receiving snapshot count
+        // TODO
+        // How many region is applying snapshot
+        // TODO
+        // When the store is started (unix timestamp in milliseconds)
+        stats.setStartTime(this.storeEngine.getStartTime());
+        // If the store is busy
+        stats.setBusy(this.storeEngine.isBusy());
+        // Actually used space by db
+        stats.setUsedSize(this.storeEngine.getStoreUsedSpace());
+        // Bytes written for the store during this period
+        stats.setBytesWritten(getStoreBytesWritten(true));
+        // Bytes read for the store during this period
+        stats.setBytesRead(getStoreBytesRead(true));
+        // Keys written for the store during this period
+        stats.setKeysWritten(getStoreKeysWritten(true));
+        // Keys read for the store during this period
+        stats.setKeysRead(getStoreKeysRead(true));
+        // Actually reported time interval
+        stats.setInterval(timeInterval);
+        LOG.info("Collect [StoreStats]: {}.", stats);
+        return stats;
+    }
+
+    public RegionStats collectRegionStats(final Region region, final TimeInterval timeInterval) {
+        final RegionStats stats = new RegionStats();
+        stats.setRegionId(region.getId());
+        // Leader Peer sending the heartbeat
+        stats.setLeader(new Peer(region.getId(), this.storeEngine.getStoreId(), this.storeEngine.getSelfEndpoint()));
+        // Leader considers that these peers are down
+        // TODO
+        // Pending peers are the peers that the leader can't consider as working followers
+        // TODO
+        // Bytes written for the region during this period
+        stats.setBytesWritten(getRegionBytesWritten(region, true));
+        // Bytes read for the region during this period
+        stats.setBytesRead(getRegionBytesRead(region, true));
+        // Keys written for the region during this period
+        stats.setKeysWritten(getRegionKeysWritten(region, true));
+        // Keys read for the region during this period
+        stats.setKeysRead(getRegionKeysRead(region, true));
+        // Approximate region size
+        // TODO very important
+        // Approximate number of keys
+        stats.setApproximateKeys(this.rawKVStore.getApproximateKeysInRange(region.getStartKey(), region.getEndKey()));
+        // Actually reported time interval
+        stats.setInterval(timeInterval);
+        LOG.info("Collect [RegionStats]: {}.", stats);
+        return stats;
+    }
+
+    public long getStoreBytesWritten(final boolean reset) {
+        if (this.rocksRawKVStore == null) {
+            return 0; // TODO memory db statistics
+        }
+        if (reset) {
+            return RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, BYTES_WRITTEN);
+        }
+        return RocksStatistics.getTickerCount(this.rocksRawKVStore, BYTES_WRITTEN);
+    }
+
+    public long getStoreBytesRead(final boolean reset) {
+        if (this.rocksRawKVStore == null) {
+            return 0; // TODO memory db statistics
+        }
+        if (reset) {
+            return RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, BYTES_READ)
+                   + RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, NUMBER_MULTIGET_BYTES_READ);
+        }
+        return RocksStatistics.getTickerCount(this.rocksRawKVStore, BYTES_READ)
+               + RocksStatistics.getTickerCount(this.rocksRawKVStore, NUMBER_MULTIGET_BYTES_READ);
+    }
+
+    public long getStoreKeysWritten(final boolean reset) {
+        if (this.rocksRawKVStore == null) {
+            return 0; // TODO memory db statistics
+        }
+        if (reset) {
+            return RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, NUMBER_KEYS_WRITTEN);
+        }
+        return RocksStatistics.getTickerCount(this.rocksRawKVStore, NUMBER_KEYS_WRITTEN);
+    }
+
+    public long getStoreKeysRead(final boolean reset) {
+        if (this.rocksRawKVStore == null) {
+            return 0; // TODO memory db statistics
+        }
+        if (reset) {
+            return RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, NUMBER_KEYS_READ)
+                   + RocksStatistics.getAndResetTickerCount(this.rocksRawKVStore, NUMBER_MULTIGET_KEYS_READ);
+        }
+        return RocksStatistics.getTickerCount(this.rocksRawKVStore, NUMBER_KEYS_READ)
+               + RocksStatistics.getTickerCount(this.rocksRawKVStore, NUMBER_MULTIGET_KEYS_READ);
+    }
+
+    public long getRegionBytesWritten(final Region region, final boolean reset) {
+        final Counter counter = KVMetrics.counter(KVMetricNames.REGION_BYTES_WRITTEN, String.valueOf(region.getId()));
+        final long value = counter.getCount();
+        if (reset) {
+            counter.dec(value);
+        }
+        return value;
+    }
+
+    public long getRegionBytesRead(final Region region, final boolean reset) {
+        final Counter counter = KVMetrics.counter(KVMetricNames.REGION_BYTES_READ, String.valueOf(region.getId()));
+        final long value = counter.getCount();
+        if (reset) {
+            counter.dec(value);
+        }
+        return value;
+    }
+
+    public long getRegionKeysWritten(final Region region, final boolean reset) {
+        final Counter counter = KVMetrics.counter(KVMetricNames.REGION_KEYS_WRITTEN, String.valueOf(region.getId()));
+        final long value = counter.getCount();
+        if (reset) {
+            counter.dec(value);
+        }
+        return value;
+    }
+
+    public long getRegionKeysRead(final Region region, final boolean reset) {
+        final Counter counter = KVMetrics.counter(KVMetricNames.REGION_KEYS_READ, String.valueOf(region.getId()));
+        final long value = counter.getCount();
+        if (reset) {
+            counter.dec(value);
+        }
+        return value;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseRequest.java
new file mode 100644
index 0000000..9c189da
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseRequest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import java.io.Serializable;
+
+/**
+ * RPC request header
+ *
+ * @author jiachun.fjc
+ */
+public abstract class BaseRequest implements Serializable {
+
+    private static final long serialVersionUID = 1056021642901412112L;
+
+    public static final byte  STORE_HEARTBEAT  = 0x01;
+    public static final byte  REGION_HEARTBEAT = 0x02;
+    public static final byte  GET_CLUSTER_INFO = 0x03;
+    public static final byte  GET_STORE_INFO   = 0x04;
+    public static final byte  SET_STORE_INFO   = 0x05;
+    public static final byte  GET_STORE_ID     = 0x06;
+    public static final byte  CREATE_REGION_ID = 0x07;
+
+    private long              clusterId;
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public void setClusterId(long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public abstract byte magic();
+
+    @Override
+    public String toString() {
+        return "BaseRequest{" + "clusterId=" + clusterId + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseResponse.java
new file mode 100644
index 0000000..b9cf72c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/BaseResponse.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+
+/**
+ * RPC response header
+ *
+ * @author jiachun.fjc
+ */
+public abstract class BaseResponse implements Serializable {
+
+    private static final long serialVersionUID = 7595480549808409921L;
+
+    private Errors            error            = Errors.NONE;
+    private long              clusterId;
+    private T                 value;
+
+    public boolean isSuccess() {
+        return error == Errors.NONE;
+    }
+
+    public Errors getError() {
+        return error;
+    }
+
+    public void setError(Errors error) {
+        this.error = error;
+    }
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public void setClusterId(long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public T getValue() {
+        return value;
+    }
+
+    public void setValue(T value) {
+        this.value = value;
+    }
+
+    @Override
+    public String toString() {
+        return "BaseResponse{" + "error=" + error + ", clusterId=" + clusterId + ", value=" + value + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdRequest.java
new file mode 100644
index 0000000..8baac06
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * @author jiachun.fjc
+ */
+public class CreateRegionIdRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -2166959562654571711L;
+
+    private Endpoint          endpoint;
+
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public void setEndpoint(Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    @Override
+    public byte magic() {
+        return CREATE_REGION_ID;
+    }
+
+    @Override
+    public String toString() {
+        return "CreateRegionIdRequest{" + "endpoint=" + endpoint + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdResponse.java
new file mode 100644
index 0000000..618afdd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/CreateRegionIdResponse.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+/**
+ * @author jiachun.fjc
+ */
+public class CreateRegionIdResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 3803254752812207217L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoRequest.java
new file mode 100644
index 0000000..9d0d6f0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoRequest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetClusterInfoRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -3037434334230192744L;
+
+    @Override
+    public byte magic() {
+        return GET_CLUSTER_INFO;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoResponse.java
new file mode 100644
index 0000000..bc764fc
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetClusterInfoResponse.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetClusterInfoResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 8811355826978037463L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdRequest.java
new file mode 100644
index 0000000..41fdc16
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetStoreIdRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 4699755250417516668L;
+
+    private Endpoint          endpoint;
+
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public void setEndpoint(Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    @Override
+    public byte magic() {
+        return GET_STORE_ID;
+    }
+
+    @Override
+    public String toString() {
+        return "GetStoreIdRequest{" + "endpoint=" + endpoint + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdResponse.java
new file mode 100644
index 0000000..ae0c442
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreIdResponse.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetStoreIdResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 1559432256700135322L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoRequest.java
new file mode 100644
index 0000000..067ba31
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetStoreInfoRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -8739048249960050226L;
+
+    private Endpoint          endpoint;
+
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public void setEndpoint(Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    @Override
+    public byte magic() {
+        return GET_STORE_INFO;
+    }
+
+    @Override
+    public String toString() {
+        return "GetStoreInfoRequest{" + "endpoint=" + endpoint + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoResponse.java
new file mode 100644
index 0000000..69533ad
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/GetStoreInfoResponse.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+
+/**
+ * @author jiachun.fjc
+ */
+public class GetStoreInfoResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 7728737397246178266L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatRequest.java
new file mode 100644
index 0000000..8356577
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatRequest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionHeartbeatRequest extends BaseRequest {
+
+    private static final long               serialVersionUID = -5149082334939576598L;
+
+    private long                            storeId;
+    private long                            leastKeysOnSplit;
+    private List> regionStatsList;
+
+    public long getStoreId() {
+        return storeId;
+    }
+
+    public void setStoreId(long storeId) {
+        this.storeId = storeId;
+    }
+
+    public long getLeastKeysOnSplit() {
+        return leastKeysOnSplit;
+    }
+
+    public void setLeastKeysOnSplit(long leastKeysOnSplit) {
+        this.leastKeysOnSplit = leastKeysOnSplit;
+    }
+
+    public List> getRegionStatsList() {
+        return regionStatsList;
+    }
+
+    public void setRegionStatsList(List> regionStatsList) {
+        this.regionStatsList = regionStatsList;
+    }
+
+    @Override
+    public byte magic() {
+        return REGION_HEARTBEAT;
+    }
+
+    @Override
+    public String toString() {
+        return "RegionHeartbeatRequest{" + "storeId=" + storeId + ", leastKeysOnSplit=" + leastKeysOnSplit
+               + ", regionStatsList=" + regionStatsList + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatResponse.java
new file mode 100644
index 0000000..97e219a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/RegionHeartbeatResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionHeartbeatResponse extends BaseResponse> {
+
+    private static final long serialVersionUID = 6562219940081433268L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoRequest.java
new file mode 100644
index 0000000..716360b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoRequest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+
+/**
+ * @author jiachun.fjc
+ */
+public class SetStoreInfoRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 6928334353055874540L;
+
+    private Store             store;
+
+    public Store getStore() {
+        return store;
+    }
+
+    public void setStore(Store store) {
+        this.store = store;
+    }
+
+    @Override
+    public byte magic() {
+        return SET_STORE_INFO;
+    }
+
+    @Override
+    public String toString() {
+        return "SetStoreInfoRequest{" + "store=" + store + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoResponse.java
new file mode 100644
index 0000000..8a8be32
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/SetStoreInfoResponse.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+
+/**
+ * @author jiachun.fjc
+ */
+public class SetStoreInfoResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -6160125980158885948L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatRequest.java
new file mode 100644
index 0000000..2ebecdf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatRequest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StoreHeartbeatRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -601174279828704074L;
+
+    private StoreStats        stats;
+
+    public StoreStats getStats() {
+        return stats;
+    }
+
+    public void setStats(StoreStats stats) {
+        this.stats = stats;
+    }
+
+    @Override
+    public byte magic() {
+        return STORE_HEARTBEAT;
+    }
+
+    @Override
+    public String toString() {
+        return "StoreHeartbeatRequest{" + "stats=" + stats + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatResponse.java
new file mode 100644
index 0000000..714e7d6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/pd/StoreHeartbeatResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.pd;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StoreHeartbeatResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -1881606836380822632L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseRequest.java
new file mode 100644
index 0000000..9d4221c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseRequest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+
+/**
+ * RPC request header
+ *
+ * @author jiachun.fjc
+ */
+public abstract class BaseRequest implements Serializable {
+
+    private static final long serialVersionUID = -6576381361684687237L;
+
+    public static final byte  PUT              = 0x01;
+    public static final byte  BATCH_PUT        = 0x02;
+    public static final byte  PUT_IF_ABSENT    = 0x03;
+    public static final byte  GET_PUT          = 0x04;
+    public static final byte  DELETE           = 0x05;
+    public static final byte  DELETE_RANGE     = 0x06;
+    public static final byte  MERGE            = 0x07;
+    public static final byte  GET              = 0x08;
+    public static final byte  MULTI_GET        = 0x09;
+    public static final byte  SCAN             = 0x0a;
+    public static final byte  GET_SEQUENCE     = 0x0b;
+    public static final byte  RESET_SEQUENCE   = 0x0c;
+    public static final byte  KEY_LOCK         = 0x0d;
+    public static final byte  KEY_UNLOCK       = 0x0e;
+    public static final byte  NODE_EXECUTE     = 0x0f;
+    public static final byte  RANGE_SPLIT      = 0x10;
+    public static final byte  COMPARE_PUT      = 0x11;
+    public static final byte  BATCH_DELETE     = 0x12;
+    public static final byte  CONTAINS_KEY     = 0x13;
+    public static final byte  COMPARE_PUT_ALL  = 0x14;
+
+    private long              regionId;
+    private RegionEpoch       regionEpoch;
+
+    public long getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(long regionId) {
+        this.regionId = regionId;
+    }
+
+    public RegionEpoch getRegionEpoch() {
+        return regionEpoch;
+    }
+
+    public void setRegionEpoch(RegionEpoch regionEpoch) {
+        this.regionEpoch = regionEpoch;
+    }
+
+    public abstract byte magic();
+
+    @Override
+    public String toString() {
+        return "BaseRequest{" + "regionId=" + regionId + ", regionEpoch=" + regionEpoch + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseResponse.java
new file mode 100644
index 0000000..a50d96f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BaseResponse.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+
+/**
+ * RPC response header
+ *
+ * @author jiachun.fjc
+ */
+public class BaseResponse implements Serializable {
+
+    private static final long serialVersionUID = 8411573936817037697L;
+
+    private Errors            error            = Errors.NONE;
+    private long              regionId;
+    private RegionEpoch       regionEpoch;
+    private T                 value;
+
+    public boolean isSuccess() {
+        return error == Errors.NONE;
+    }
+
+    public Errors getError() {
+        return error;
+    }
+
+    public void setError(Errors error) {
+        this.error = error;
+    }
+
+    public long getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(long regionId) {
+        this.regionId = regionId;
+    }
+
+    public RegionEpoch getRegionEpoch() {
+        return regionEpoch;
+    }
+
+    public void setRegionEpoch(RegionEpoch regionEpoch) {
+        this.regionEpoch = regionEpoch;
+    }
+
+    public T getValue() {
+        return value;
+    }
+
+    public void setValue(T value) {
+        this.value = value;
+    }
+
+    @Override
+    public String toString() {
+        return "BaseResponse{" + "error=" + error + ", regionId=" + regionId + ", regionEpoch=" + regionEpoch
+               + ", value=" + value + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteRequest.java
new file mode 100644
index 0000000..7620ea6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteRequest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.List;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class BatchDeleteRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -472951628397420368L;
+
+    private List      keys;
+
+    public List getKeys() {
+        return keys;
+    }
+
+    public void setKeys(List keys) {
+        this.keys = keys;
+    }
+
+    @Override
+    public byte magic() {
+        return BATCH_DELETE;
+    }
+
+    @Override
+    public String toString() {
+        return "BatchDeleteRequest{" + "keys=" + keys + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteResponse.java
new file mode 100644
index 0000000..55aef68
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchDeleteResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class BatchDeleteResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 8507492618405370694L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutRequest.java
new file mode 100644
index 0000000..6eec7aa
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutRequest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BatchPutRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -980036845124180958L;
+
+    private List     kvEntries;
+
+    public List getKvEntries() {
+        return kvEntries;
+    }
+
+    public void setKvEntries(List kvEntries) {
+        this.kvEntries = kvEntries;
+    }
+
+    @Override
+    public byte magic() {
+        return BATCH_PUT;
+    }
+
+    @Override
+    public String toString() {
+        return "BatchPutRequest{" + "kvEntries=" + kvEntries + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutResponse.java
new file mode 100644
index 0000000..2bef034
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/BatchPutResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BatchPutResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 5567277378042784855L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllRequest.java
new file mode 100644
index 0000000..f54ddbc
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllRequest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class CASAllRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -241778901444215595L;
+
+    private List    casEntries;
+
+    public List getCasEntries() {
+        return casEntries;
+    }
+
+    public void setCasEntries(List casEntries) {
+        this.casEntries = casEntries;
+    }
+
+    @Override
+    public byte magic() {
+        return COMPARE_PUT_ALL;
+    }
+
+    @Override
+    public String toString() {
+        return "CASAllRequest{" + "casEntries=" + casEntries + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllResponse.java
new file mode 100644
index 0000000..bff80aa
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CASAllResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class CASAllResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -3264033362512003699L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutRequest.java
new file mode 100644
index 0000000..9ec7e70
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutRequest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class CompareAndPutRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -6395140862740192718L;
+
+    private byte[]            key;
+    private byte[]            expect;
+    private byte[]            update;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getExpect() {
+        return expect;
+    }
+
+    public void setExpect(byte[] expect) {
+        this.expect = expect;
+    }
+
+    public byte[] getUpdate() {
+        return update;
+    }
+
+    public void setUpdate(byte[] update) {
+        this.update = update;
+    }
+
+    @Override
+    public byte magic() {
+        return COMPARE_PUT;
+    }
+
+    @Override
+    public String toString() {
+        return "CompareAndPutRequest{" + "key=" + BytesUtil.toHex(key) + ", expect=" + BytesUtil.toHex(expect)
+               + ", update=" + BytesUtil.toHex(update) + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutResponse.java
new file mode 100644
index 0000000..f4719b0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/CompareAndPutResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class CompareAndPutResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -4720936140914607358L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyRequest.java
new file mode 100644
index 0000000..46236ce
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyRequest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class ContainsKeyRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -2690374162748351638L;
+
+    private byte[]            key;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    @Override
+    public byte magic() {
+        return CONTAINS_KEY;
+    }
+
+    @Override
+    public String toString() {
+        return "ContainsKeyRequest{" + "key=" + BytesUtil.toHex(key) + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyResponse.java
new file mode 100644
index 0000000..0169e26
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ContainsKeyResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author nicholas.jxf
+ */
+public class ContainsKeyResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -6915749263924784257L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeRequest.java
new file mode 100644
index 0000000..907a2d0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteRangeRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -5457684744947827779L;
+
+    private byte[]            startKey;
+    private byte[]            endKey;
+
+    public byte[] getStartKey() {
+        return startKey;
+    }
+
+    public void setStartKey(byte[] startKey) {
+        this.startKey = startKey;
+    }
+
+    public byte[] getEndKey() {
+        return endKey;
+    }
+
+    public void setEndKey(byte[] endKey) {
+        this.endKey = endKey;
+    }
+
+    @Override
+    public byte magic() {
+        return DELETE_RANGE;
+    }
+
+    @Override
+    public String toString() {
+        return "DeleteRangeRequest{" + "startKey=" + BytesUtil.toHex(startKey) + ", endKey=" + BytesUtil.toHex(endKey)
+               + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeResponse.java
new file mode 100644
index 0000000..931f771
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRangeResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteRangeResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 8072092782107062783L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRequest.java
new file mode 100644
index 0000000..3dc72da
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteRequest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -6230976612126054651L;
+
+    private byte[]            key;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    @Override
+    public byte magic() {
+        return DELETE;
+    }
+
+    @Override
+    public String toString() {
+        return "DeleteRequest{" + "key=" + BytesUtil.toHex(key) + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteResponse.java
new file mode 100644
index 0000000..d5fbe29
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/DeleteResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DeleteResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 660998710428573132L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutRequest.java
new file mode 100644
index 0000000..bd789b8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetAndPutRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -9141052543409768050L;
+
+    private byte[]            key;
+    private byte[]            value;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+
+    @Override
+    public byte magic() {
+        return GET_PUT;
+    }
+
+    @Override
+    public String toString() {
+        return "GetAndPutRequest{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + "} "
+               + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutResponse.java
new file mode 100644
index 0000000..61083a8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetAndPutResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetAndPutResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -2858497754385528403L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetRequest.java
new file mode 100644
index 0000000..6c9e725
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -864939889102077919L;
+
+    private byte[]            key;
+    private boolean           readOnlySafe     = true;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public boolean isReadOnlySafe() {
+        return readOnlySafe;
+    }
+
+    public void setReadOnlySafe(boolean readOnlySafe) {
+        this.readOnlySafe = readOnlySafe;
+    }
+
+    @Override
+    public byte magic() {
+        return GET;
+    }
+
+    @Override
+    public String toString() {
+        return "GetRequest{" + "key=" + BytesUtil.toHex(key) + ", readOnlySafe=" + readOnlySafe + "} "
+               + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetResponse.java
new file mode 100644
index 0000000..0a2ec69
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -1352343281441520063L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceRequest.java
new file mode 100644
index 0000000..26a8888
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceRequest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetSequenceRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 8409861577121335137L;
+
+    private byte[]            seqKey;
+    private int               step;
+
+    public byte[] getSeqKey() {
+        return seqKey;
+    }
+
+    public void setSeqKey(byte[] seqKey) {
+        this.seqKey = seqKey;
+    }
+
+    public int getStep() {
+        return step;
+    }
+
+    public void setStep(int step) {
+        this.step = step;
+    }
+
+    @Override
+    public byte magic() {
+        return GET_SEQUENCE;
+    }
+
+    @Override
+    public String toString() {
+        return "GetSequenceRequest{" + "seqKey=" + BytesUtil.toHex(seqKey) + ", step=" + step + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceResponse.java
new file mode 100644
index 0000000..75c1c47
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/GetSequenceResponse.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class GetSequenceResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 4145243784727963679L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockRequest.java
new file mode 100644
index 0000000..0071570
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockRequest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class KeyLockRequest extends BaseRequest {
+
+    private static final long        serialVersionUID = -2378291429837556797L;
+
+    private byte[]                   key;
+    private boolean                  keepLease;
+    private DistributedLock.Acquirer acquirer;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public boolean isKeepLease() {
+        return keepLease;
+    }
+
+    public void setKeepLease(boolean keepLease) {
+        this.keepLease = keepLease;
+    }
+
+    public DistributedLock.Acquirer getAcquirer() {
+        return acquirer;
+    }
+
+    public void setAcquirer(DistributedLock.Acquirer acquirer) {
+        this.acquirer = acquirer;
+    }
+
+    @Override
+    public byte magic() {
+        return KEY_LOCK;
+    }
+
+    @Override
+    public String toString() {
+        return "KeyLockRequest{" + "key=" + BytesUtil.toHex(key) + ", keepLease=" + keepLease + ", acquirer="
+               + acquirer + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockResponse.java
new file mode 100644
index 0000000..ee4a223
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyLockResponse.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class KeyLockResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -4882173656568408696L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockRequest.java
new file mode 100644
index 0000000..600d829
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class KeyUnlockRequest extends BaseRequest {
+
+    private static final long        serialVersionUID = 4301415966181665575L;
+
+    private byte[]                   key;
+    private DistributedLock.Acquirer acquirer;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public DistributedLock.Acquirer getAcquirer() {
+        return acquirer;
+    }
+
+    public void setAcquirer(DistributedLock.Acquirer acquirer) {
+        this.acquirer = acquirer;
+    }
+
+    @Override
+    public byte magic() {
+        return KEY_UNLOCK;
+    }
+
+    @Override
+    public String toString() {
+        return "KeyUnlockRequest{" + "key=" + BytesUtil.toHex(key) + ", acquirer=" + acquirer + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockResponse.java
new file mode 100644
index 0000000..4b7aa53
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/KeyUnlockResponse.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class KeyUnlockResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 1062747000551501541L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeRequest.java
new file mode 100644
index 0000000..9c84ce6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MergeRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -1438709327379459775L;
+
+    private byte[]            key;
+    private byte[]            value;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+
+    @Override
+    public byte magic() {
+        return MERGE;
+    }
+
+    @Override
+    public String toString() {
+        return "MergeRequest{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + "} "
+               + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeResponse.java
new file mode 100644
index 0000000..6e9e97d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MergeResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MergeResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 3276165641127952711L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetRequest.java
new file mode 100644
index 0000000..86f5b1b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetRequest.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.List;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MultiGetRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 586927459257448933L;
+
+    private List      keys;
+    private boolean           readOnlySafe     = true;
+
+    public List getKeys() {
+        return keys;
+    }
+
+    public void setKeys(List keys) {
+        this.keys = keys;
+    }
+
+    public boolean isReadOnlySafe() {
+        return readOnlySafe;
+    }
+
+    public void setReadOnlySafe(boolean readOnlySafe) {
+        this.readOnlySafe = readOnlySafe;
+    }
+
+    @Override
+    public byte magic() {
+        return MULTI_GET;
+    }
+
+    @Override
+    public String toString() {
+        return "MultiGetRequest{" + "keys=" + keys + ", readOnlySafe=" + readOnlySafe + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetResponse.java
new file mode 100644
index 0000000..b27d998
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/MultiGetResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.Map;
+
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MultiGetResponse extends BaseResponse> {
+
+    private static final long serialVersionUID = -7136277297323938853L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NoRegionFoundResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NoRegionFoundResponse.java
new file mode 100644
index 0000000..1e67c3c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NoRegionFoundResponse.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ * Region does not exist in the current node.
+ *
+ * @author jiachun.fjc
+ */
+public class NoRegionFoundResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -3274115927148766255L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteRequest.java
new file mode 100644
index 0000000..31cc891
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteRequest.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.rhea.storage.NodeExecutor;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class NodeExecuteRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -9087272065211874819L;
+
+    private NodeExecutor      nodeExecutor;
+
+    public NodeExecutor getNodeExecutor() {
+        return nodeExecutor;
+    }
+
+    public void setNodeExecutor(NodeExecutor nodeExecutor) {
+        this.nodeExecutor = nodeExecutor;
+    }
+
+    @Override
+    public byte magic() {
+        return NODE_EXECUTE;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteResponse.java
new file mode 100644
index 0000000..787871e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/NodeExecuteResponse.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ * @author jiachun.fjc
+ */
+public class NodeExecuteResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -6720048151703733047L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentRequest.java
new file mode 100644
index 0000000..aa64f3f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutIfAbsentRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 5937066725083445707L;
+
+    private byte[]            key;
+    private byte[]            value;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+
+    @Override
+    public byte magic() {
+        return PUT_IF_ABSENT;
+    }
+
+    @Override
+    public String toString() {
+        return "PutRequest{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + "} "
+               + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentResponse.java
new file mode 100644
index 0000000..1975b65
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutIfAbsentResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutIfAbsentResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 4041608778755970883L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutRequest.java
new file mode 100644
index 0000000..76529e2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 5937066725083445707L;
+
+    private byte[]            key;
+    private byte[]            value;
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+
+    @Override
+    public byte magic() {
+        return PUT;
+    }
+
+    @Override
+    public String toString() {
+        return "PutRequest{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + "} "
+               + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutResponse.java
new file mode 100644
index 0000000..ac86e8f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/PutResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PutResponse extends BaseResponse {
+
+    private static final long serialVersionUID = 4041608778755970883L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitRequest.java
new file mode 100644
index 0000000..053133c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitRequest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RangeSplitRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 2369343322478279224L;
+
+    private Long              newRegionId;
+
+    public Long getNewRegionId() {
+        return newRegionId;
+    }
+
+    public void setNewRegionId(Long newRegionId) {
+        this.newRegionId = newRegionId;
+    }
+
+    @Override
+    public byte magic() {
+        return RANGE_SPLIT;
+    }
+
+    @Override
+    public String toString() {
+        return "RangeSplitRequest{" + "newRegionId=" + newRegionId + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitResponse.java
new file mode 100644
index 0000000..d733c7b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/RangeSplitResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RangeSplitResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -2041687876902891674L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceRequest.java
new file mode 100644
index 0000000..5520650
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceRequest.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ResetSequenceRequest extends BaseRequest {
+
+    private static final long serialVersionUID = 9173433364010147694L;
+
+    private byte[]            seqKey;
+
+    public byte[] getSeqKey() {
+        return seqKey;
+    }
+
+    public void setSeqKey(byte[] seqKey) {
+        this.seqKey = seqKey;
+    }
+
+    @Override
+    public byte magic() {
+        return RESET_SEQUENCE;
+    }
+
+    @Override
+    public String toString() {
+        return "ResetSequenceRequest{" + "seqKey=" + BytesUtil.toHex(seqKey) + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceResponse.java
new file mode 100644
index 0000000..19c3a37
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ResetSequenceResponse.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ResetSequenceResponse extends BaseResponse {
+
+    private static final long serialVersionUID = -4218451773245085392L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanRequest.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanRequest.java
new file mode 100644
index 0000000..0e058e6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanRequest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ScanRequest extends BaseRequest {
+
+    private static final long serialVersionUID = -7229126480671449199L;
+
+    private byte[]            startKey;
+    private byte[]            endKey;
+    // If limit == 0, it will be modified to Integer.MAX_VALUE on the server
+    // and then queried.  So 'limit == 0' means that the number of queries is
+    // not limited. This is because serialization uses varint to compress
+    // numbers.  In the case of 0, only 1 byte is occupied, and Integer.MAX_VALUE
+    // takes 5 bytes.
+    private int               limit;
+    private boolean           readOnlySafe     = true;
+    private boolean           returnValue      = true;
+    private boolean           reverse          = false;
+
+    public boolean isReverse() {
+        return reverse;
+    }
+
+    public void setReverse(boolean reverse) {
+        this.reverse = reverse;
+    }
+
+    public byte[] getStartKey() {
+        return startKey;
+    }
+
+    public void setStartKey(byte[] startKey) {
+        this.startKey = startKey;
+    }
+
+    public byte[] getEndKey() {
+        return endKey;
+    }
+
+    public void setEndKey(byte[] endKey) {
+        this.endKey = endKey;
+    }
+
+    public int getLimit() {
+        return limit;
+    }
+
+    public void setLimit(int limit) {
+        this.limit = limit;
+    }
+
+    public boolean isReadOnlySafe() {
+        return readOnlySafe;
+    }
+
+    public void setReadOnlySafe(boolean readOnlySafe) {
+        this.readOnlySafe = readOnlySafe;
+    }
+
+    public boolean isReturnValue() {
+        return returnValue;
+    }
+
+    public void setReturnValue(boolean returnValue) {
+        this.returnValue = returnValue;
+    }
+
+    @Override
+    public byte magic() {
+        return SCAN;
+    }
+
+    @Override
+    public String toString() {
+        return "ScanRequest{" + "startKey=" + BytesUtil.toHex(startKey) + ", endKey=" + BytesUtil.toHex(endKey)
+               + ", limit=" + limit + ", reverse=" + reverse + ", readOnlySafe=" + readOnlySafe + ", returnValue="
+               + returnValue + "} " + super.toString();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanResponse.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanResponse.java
new file mode 100644
index 0000000..c004588
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/cmd/store/ScanResponse.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.cmd.store;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ScanResponse extends BaseResponse> {
+
+    private static final long serialVersionUID = 4993114150396497063L;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiException.java
new file mode 100644
index 0000000..5bcb1f5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Any API exception that is part of the public protocol
+ */
+public class ApiException extends RuntimeException {
+
+    private static final long serialVersionUID = 1L;
+
+    public ApiException() {
+    }
+
+    public ApiException(String message) {
+        super(message);
+    }
+
+    public ApiException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ApiException(Throwable cause) {
+        super(cause);
+    }
+
+    /* avoid the expensive and useless stack trace for api exceptions */
+    @Override
+    public Throwable fillInStackTrace() {
+        return this;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiExceptionHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiExceptionHelper.java
new file mode 100644
index 0000000..10413ca
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ApiExceptionHelper.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class ApiExceptionHelper {
+
+    // require refresh region route table
+    public static boolean isInvalidEpoch(final Throwable cause) {
+        return cause instanceof InvalidRegionMembershipException //
+               || cause instanceof InvalidRegionVersionException //
+               || cause instanceof InvalidRegionEpochException;
+    }
+
+    private ApiExceptionHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/CallSelfEndpointException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/CallSelfEndpointException.java
new file mode 100644
index 0000000..4635be7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/CallSelfEndpointException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class CallSelfEndpointException extends ApiException {
+
+    private static final long serialVersionUID = -5346262835802551098L;
+
+    public CallSelfEndpointException() {
+    }
+
+    public CallSelfEndpointException(String message) {
+        super(message);
+    }
+
+    public CallSelfEndpointException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public CallSelfEndpointException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/Errors.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/Errors.java
new file mode 100644
index 0000000..c8f92ee
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/Errors.java
@@ -0,0 +1,206 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+
+/**
+ * This class contains all the client-server errors--those errors that must be sent from the server to the client. These
+ * are thus part of the protocol. The names can be changed but the error code cannot.
+ *
+ * Note that client library will convert an unknown error code to the non-retriable UnknownServerException if the client library
+ * version is old and does not recognize the newly-added error code. Therefore when a new server-side error is added,
+ * we may need extra logic to convert the new error code to another existing error code before sending the response back to
+ * the client if the request version suggests that the client may not recognize the new error code.
+ *
+ * Do not add exceptions that occur only on the client or only on the server here.
+ */
+public enum Errors {
+    UNKNOWN_SERVER_ERROR(-1, "The server experienced an unexpected error when processing the request",
+        UnknownServerException::new),
+
+    NONE(0, null, message -> null),
+
+    STORAGE_ERROR(1, "Disk error when trying to access log file on the disk.", StorageException::new),
+
+    INVALID_REQUEST(2, "This most likely occurs because of a request being malformed by the "
+                       + "client library or the message was sent to an incompatible server. See the server logs "
+                       + "for more details.", InvalidRequestException::new),
+
+    LEADER_NOT_AVAILABLE(3, "The leader is not available.", LeaderNotAvailableException::new),
+
+    NOT_LEADER(4, "This is not the correct leader.", NotLeaderException::new),
+
+    INVALID_PARAMETER(5, "Invalid parameter error, please check your input parameters. See the server logs for more "
+                         + "details.", InvalidParameterException::new),
+
+    NO_REGION_FOUND(6, "Can not find region error.", NoRegionFoundException::new),
+
+    INVALID_REGION_MEMBERSHIP(7, "Invalid region membership config error (add or remove peer happened).",
+        InvalidRegionMembershipException::new),
+
+    INVALID_REGION_VERSION(8, "Invalid region version error (region split or merge happened).",
+        InvalidRegionVersionException::new),
+
+    INVALID_REGION_EPOCH(9, "Invalid region epoch (membership or version changed).", InvalidRegionEpochException::new),
+
+    INVALID_STORE_STATS(10, "Placement driver: invalid store stats", InvalidStoreStatsException::new),
+
+    INVALID_REGION_STATS(11, "Placement driver: invalid region stats", InvalidStoreStatsException::new),
+
+    STORE_HEARTBEAT_OUT_OF_DATE(12, "The store heartbeat info is out of date", StoreHeartbeatOutOfDateException::new),
+
+    REGION_HEARTBEAT_OUT_OF_DATE(13, "The region heartbeat info is out of date", RegionHeartbeatOutOfDateException::new),
+
+    CALL_SELF_ENDPOINT_ERROR(14, "The usual reason is that the rpc call selected the self endpoint.",
+        CallSelfEndpointException::new),
+
+    SERVER_BUSY(15, "The server is busy now.", ServerBusyException::new),
+
+    REGION_ENGINE_FAIL(16, "Fail to start region engine. See the server logs for more details.",
+        RegionEngineFailException::new),
+
+    CONFLICT_REGION_ID(17, "There is a conflict between the new region id and the existing ids. "
+                            + "The new region cannot be created.", RangeSplitFailException::new),
+
+    TOO_SMALL_TO_SPLIT(18, "The region size is too small to split. See the server logs for more details.",
+        RangeSplitFailException::new);
+
+    private interface ApiExceptionBuilder {
+        ApiException build(final String message);
+    }
+
+    private static final Logger          LOG          = LoggerFactory.getLogger(Errors.class);
+
+    private static Map, Errors> classToError = Maps.newHashMap();
+    private static Map    codeToError  = Maps.newHashMap();
+
+    static {
+        for (final Errors error : Errors.values()) {
+            codeToError.put(error.code(), error);
+            if (error.exception != null) {
+                classToError.put(error.exception.getClass(), error);
+            }
+        }
+    }
+
+    private final short                  code;
+    private final ApiExceptionBuilder    builder;
+    private final ApiException           exception;
+
+    Errors(int code, String defaultExceptionString, ApiExceptionBuilder builder) {
+        this.code = (short) code;
+        this.builder = builder;
+        this.exception = builder.build(defaultExceptionString);
+    }
+
+    /**
+     * An instance of the exception
+     */
+    public ApiException exception() {
+        return this.exception;
+    }
+
+    /**
+     * Create an instance of the ApiException that contains the given error message.
+     *
+     * @param message The message string to set.
+     * @return        The exception.
+     */
+    public ApiException exception(final String message) {
+        if (message == null) {
+            // If no error message was specified, return an exception with the default error message.
+            return this.exception;
+        }
+        // Return an exception with the given error message.
+        return this.builder.build(message);
+    }
+
+    /**
+     * Returns the class name of the exception or null if this is {@code Errors.NONE}.
+     */
+    public String exceptionName() {
+        return this.exception == null ? null : this.exception.getClass().getName();
+    }
+
+    /**
+     * The error code for the exception
+     */
+    public short code() {
+        return this.code;
+    }
+
+    /**
+     * Throw the exception corresponding to this error if there is one
+     */
+    public void maybeThrow() {
+        if (this.exception != null) {
+            throw this.exception;
+        }
+    }
+
+    /**
+     * Get a friendly description of the error (if one is available).
+     * @return the error message
+     */
+    public String message() {
+        if (this.exception != null) {
+            return this.exception.getMessage();
+        }
+        return toString();
+    }
+
+    /**
+     * Throw the exception if there is one
+     */
+    public static Errors forCode(final short code) {
+        final Errors error = codeToError.get(code);
+        if (error != null) {
+            return error;
+        } else {
+            LOG.error("Unexpected error code: {}.", code);
+            return UNKNOWN_SERVER_ERROR;
+        }
+    }
+
+    /**
+     * Return the error instance associated with this exception or any of its superclasses (or UNKNOWN if there is none).
+     * If there are multiple matches in the class hierarchy, the first match starting from the bottom is used.
+     */
+    public static Errors forException(final Throwable t) {
+        Class clazz = t.getClass();
+        while (clazz != null) {
+            final Errors error = classToError.get(clazz);
+            if (error != null) {
+                return error;
+            }
+            clazz = clazz.getSuperclass();
+        }
+        LOG.error("Unexpected error: {}.", StackTraceUtil.stackTrace(t));
+        return UNKNOWN_SERVER_ERROR;
+    }
+
+    public boolean isSuccess() {
+        return this == Errors.NONE;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ErrorsHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ErrorsHelper.java
new file mode 100644
index 0000000..88e1edb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ErrorsHelper.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class ErrorsHelper {
+
+    // require refresh leader or peer
+    public static boolean isInvalidPeer(final Errors error) {
+        return error == Errors.CALL_SELF_ENDPOINT_ERROR //
+               || error == Errors.NOT_LEADER //
+               || error == Errors.NO_REGION_FOUND //
+               || error == Errors.LEADER_NOT_AVAILABLE;
+    }
+
+    // require refresh region route table
+    public static boolean isInvalidEpoch(final Errors error) {
+        return error == Errors.INVALID_REGION_MEMBERSHIP //
+               || error == Errors.INVALID_REGION_VERSION //
+               || error == Errors.INVALID_REGION_EPOCH;
+    }
+
+    private ErrorsHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/IllegalKVOperationException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/IllegalKVOperationException.java
new file mode 100644
index 0000000..5a8c217
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/IllegalKVOperationException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class IllegalKVOperationException extends RheaRuntimeException {
+
+    private static final long serialVersionUID = 6265505295002925357L;
+
+    public IllegalKVOperationException() {
+    }
+
+    public IllegalKVOperationException(String message) {
+        super(message);
+    }
+
+    public IllegalKVOperationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public IllegalKVOperationException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidIteratorVersion.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidIteratorVersion.java
new file mode 100644
index 0000000..326e838
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidIteratorVersion.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class InvalidIteratorVersion extends RuntimeException {
+
+    private static final long serialVersionUID = 5867777305281344245L;
+
+    public InvalidIteratorVersion() {
+    }
+
+    public InvalidIteratorVersion(String message) {
+        super(message);
+    }
+
+    public InvalidIteratorVersion(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidIteratorVersion(Throwable cause) {
+        super(cause);
+    }
+
+    public InvalidIteratorVersion(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidLockAcquirerException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidLockAcquirerException.java
new file mode 100644
index 0000000..5853241
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidLockAcquirerException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class InvalidLockAcquirerException extends IllegalStateException {
+
+    private static final long serialVersionUID = 2699110936640666381L;
+
+    public InvalidLockAcquirerException() {
+    }
+
+    public InvalidLockAcquirerException(String s) {
+        super(s);
+    }
+
+    public InvalidLockAcquirerException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidLockAcquirerException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidMetadataException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidMetadataException.java
new file mode 100644
index 0000000..29adfbf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidMetadataException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * An exception that may indicate the client's metadata is out of date
+ */
+public abstract class InvalidMetadataException extends RetriableException {
+
+    private static final long serialVersionUID = 1L;
+
+    public InvalidMetadataException() {
+        super();
+    }
+
+    public InvalidMetadataException(String message) {
+        super(message);
+    }
+
+    public InvalidMetadataException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidMetadataException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidParameterException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidParameterException.java
new file mode 100644
index 0000000..db19a5d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidParameterException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Invalid input parameter error.
+ *
+ * @author jiachun.fjc
+ */
+public class InvalidParameterException extends ApiException {
+
+    private static final long serialVersionUID = 1571203917022285992L;
+
+    public InvalidParameterException() {
+    }
+
+    public InvalidParameterException(String message) {
+        super(message);
+    }
+
+    public InvalidParameterException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidParameterException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionEpochException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionEpochException.java
new file mode 100644
index 0000000..cea1dc8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionEpochException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class InvalidRegionEpochException extends ApiException {
+
+    private static final long serialVersionUID = -6773905397707597679L;
+
+    public InvalidRegionEpochException() {
+    }
+
+    public InvalidRegionEpochException(String message) {
+        super(message);
+    }
+
+    public InvalidRegionEpochException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidRegionEpochException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionMembershipException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionMembershipException.java
new file mode 100644
index 0000000..59a136e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionMembershipException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Membership changes occur in the current region
+ *
+ * @author jiachun.fjc
+ */
+public class InvalidRegionMembershipException extends ApiException {
+
+    private static final long serialVersionUID = 568193660861704723L;
+
+    public InvalidRegionMembershipException() {
+    }
+
+    public InvalidRegionMembershipException(String message) {
+        super(message);
+    }
+
+    public InvalidRegionMembershipException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidRegionMembershipException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionStatsException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionStatsException.java
new file mode 100644
index 0000000..38b59ea
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionStatsException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class InvalidRegionStatsException extends ApiException {
+
+    private static final long serialVersionUID = -7345366727426726056L;
+
+    public InvalidRegionStatsException() {
+    }
+
+    public InvalidRegionStatsException(String message) {
+        super(message);
+    }
+
+    public InvalidRegionStatsException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidRegionStatsException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionVersionException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionVersionException.java
new file mode 100644
index 0000000..79230ba
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRegionVersionException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * A split occurs in the region.
+ *
+ * @author jiachun.fjc
+ */
+public class InvalidRegionVersionException extends ApiException {
+
+    private static final long serialVersionUID = 1571203917022285992L;
+
+    public InvalidRegionVersionException() {
+    }
+
+    public InvalidRegionVersionException(String message) {
+        super(message);
+    }
+
+    public InvalidRegionVersionException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidRegionVersionException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRequestException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRequestException.java
new file mode 100644
index 0000000..d6e9974
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidRequestException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Thrown when a request breaks basic wire protocol rules.
+ * This most likely occurs because of a request being malformed by the client library or
+ * the message was sent to an incompatible broker.
+ */
+public class InvalidRequestException extends ApiException {
+
+    private static final long serialVersionUID = 1L;
+
+    public InvalidRequestException(String message) {
+        super(message);
+    }
+
+    public InvalidRequestException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidStoreStatsException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidStoreStatsException.java
new file mode 100644
index 0000000..d051d73
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/InvalidStoreStatsException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class InvalidStoreStatsException extends ApiException {
+
+    private static final long serialVersionUID = 4575768421994347194L;
+
+    public InvalidStoreStatsException() {
+    }
+
+    public InvalidStoreStatsException(String message) {
+        super(message);
+    }
+
+    public InvalidStoreStatsException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public InvalidStoreStatsException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/LeaderNotAvailableException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/LeaderNotAvailableException.java
new file mode 100644
index 0000000..7bb84c1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/LeaderNotAvailableException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * There is no currently available leader for the given partition (either because a leadership election is in progress
+ * or because all replicas are down).
+ */
+public class LeaderNotAvailableException extends InvalidMetadataException {
+
+    private static final long serialVersionUID = 1L;
+
+    public LeaderNotAvailableException(String message) {
+        super(message);
+    }
+
+    public LeaderNotAvailableException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NeverGetHereException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NeverGetHereException.java
new file mode 100644
index 0000000..d7bf93c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NeverGetHereException.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class NeverGetHereException extends RuntimeException {
+
+    private static final long                 serialVersionUID = 1526796993531347794L;
+
+    public static final NeverGetHereException INSTANCE         = new NeverGetHereException();
+
+    private NeverGetHereException() {
+    }
+
+    @Override
+    public Throwable fillInStackTrace() {
+        return this;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NoRegionFoundException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NoRegionFoundException.java
new file mode 100644
index 0000000..f59a83e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NoRegionFoundException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class NoRegionFoundException extends ApiException {
+
+    private static final long serialVersionUID = 1284652280085988572L;
+
+    public NoRegionFoundException() {
+    }
+
+    public NoRegionFoundException(String message) {
+        super(message);
+    }
+
+    public NoRegionFoundException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public NoRegionFoundException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NotLeaderException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NotLeaderException.java
new file mode 100644
index 0000000..4f67e74
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/NotLeaderException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+public class NotLeaderException extends RetriableException {
+
+    private static final long serialVersionUID = 1L;
+
+    public NotLeaderException(String message) {
+        super(message);
+    }
+
+    public NotLeaderException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RangeSplitFailException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RangeSplitFailException.java
new file mode 100644
index 0000000..023f119
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RangeSplitFailException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RangeSplitFailException extends ApiException {
+
+    private static final long serialVersionUID = -5958618049335988246L;
+
+    public RangeSplitFailException() {
+    }
+
+    public RangeSplitFailException(String message) {
+        super(message);
+    }
+
+    public RangeSplitFailException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RangeSplitFailException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionEngineFailException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionEngineFailException.java
new file mode 100644
index 0000000..e87c4bd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionEngineFailException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionEngineFailException extends ApiException {
+
+    private static final long serialVersionUID = -1951306006419415048L;
+
+    public RegionEngineFailException() {
+    }
+
+    public RegionEngineFailException(String message) {
+        super(message);
+    }
+
+    public RegionEngineFailException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RegionEngineFailException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionHeartbeatOutOfDateException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionHeartbeatOutOfDateException.java
new file mode 100644
index 0000000..4c2c8ef
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RegionHeartbeatOutOfDateException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionHeartbeatOutOfDateException extends ApiException {
+
+    private static final long serialVersionUID = -1214148060593528609L;
+
+    public RegionHeartbeatOutOfDateException() {
+    }
+
+    public RegionHeartbeatOutOfDateException(String message) {
+        super(message);
+    }
+
+    public RegionHeartbeatOutOfDateException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RegionHeartbeatOutOfDateException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RetriableException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RetriableException.java
new file mode 100644
index 0000000..38843f2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RetriableException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * A retryable exception is a transient exception that if retried may succeed.
+ */
+public abstract class RetriableException extends ApiException {
+
+    private static final long serialVersionUID = 1L;
+
+    public RetriableException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RetriableException(String message) {
+        super(message);
+    }
+
+    public RetriableException(Throwable cause) {
+        super(cause);
+    }
+
+    public RetriableException() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RheaRuntimeException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RheaRuntimeException.java
new file mode 100644
index 0000000..69b90d1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RheaRuntimeException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaRuntimeException extends RuntimeException {
+
+    private static final long serialVersionUID = 1L;
+
+    public RheaRuntimeException() {
+    }
+
+    public RheaRuntimeException(String message) {
+        super(message);
+    }
+
+    public RheaRuntimeException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RheaRuntimeException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RouteTableException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RouteTableException.java
new file mode 100644
index 0000000..a3206c2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/RouteTableException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Routing table information get/refresh failed.
+ *
+ * @author jiachun.fjc
+ */
+public class RouteTableException extends RuntimeException {
+
+    private static final long serialVersionUID = 7286710905394365815L;
+
+    public RouteTableException() {
+    }
+
+    public RouteTableException(String message) {
+        super(message);
+    }
+
+    public RouteTableException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public RouteTableException(Throwable cause) {
+        super(cause);
+    }
+
+    public RouteTableException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ServerBusyException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ServerBusyException.java
new file mode 100644
index 0000000..21d6a2e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/ServerBusyException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ServerBusyException extends ApiException {
+
+    private static final long serialVersionUID = -8818890748531018697L;
+
+    public ServerBusyException() {
+    }
+
+    public ServerBusyException(String message) {
+        super(message);
+    }
+
+    public ServerBusyException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public ServerBusyException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StorageException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StorageException.java
new file mode 100644
index 0000000..3b3754f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StorageException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StorageException extends ApiException {
+
+    private static final long serialVersionUID = 1L;
+
+    public StorageException() {
+        super();
+    }
+
+    public StorageException(String message) {
+        super(message);
+    }
+
+    public StorageException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public StorageException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreCodecException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreCodecException.java
new file mode 100644
index 0000000..cbaa743
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreCodecException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * Storage codec exception.
+ *
+ * @author dennis
+ */
+public class StoreCodecException extends RheaRuntimeException {
+
+    private static final long serialVersionUID = -8728851683513355905L;
+
+    public StoreCodecException() {
+    }
+
+    public StoreCodecException(String message) {
+        super(message);
+    }
+
+    public StoreCodecException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public StoreCodecException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreHeartbeatOutOfDateException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreHeartbeatOutOfDateException.java
new file mode 100644
index 0000000..acb3a31
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/StoreHeartbeatOutOfDateException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StoreHeartbeatOutOfDateException extends ApiException {
+
+    private static final long serialVersionUID = -6941357836000397648L;
+
+    public StoreHeartbeatOutOfDateException() {
+    }
+
+    public StoreHeartbeatOutOfDateException(String message) {
+        super(message);
+    }
+
+    public StoreHeartbeatOutOfDateException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public StoreHeartbeatOutOfDateException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/UnknownServerException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/UnknownServerException.java
new file mode 100644
index 0000000..15c490a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/errors/UnknownServerException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * An error occurred on the server for which the client doesn't have a corresponding error code. This is generally an
+ * unexpected error.
+ * 
+ */
+public class UnknownServerException extends ApiException {
+
+    private static final long serialVersionUID = 1L;
+
+    public UnknownServerException() {
+    }
+
+    public UnknownServerException(String message) {
+        super(message);
+    }
+
+    public UnknownServerException(Throwable cause) {
+        super(cause);
+    }
+
+    public UnknownServerException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Cluster.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Cluster.java
new file mode 100644
index 0000000..661845f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Cluster.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.Copiable;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Cluster implements Copiable, Serializable {
+
+    private static final long serialVersionUID = 3291666486933960310L;
+
+    private long              clusterId;
+    private List       stores;
+
+    public Cluster(long clusterId, List stores) {
+        this.clusterId = clusterId;
+        this.stores = stores;
+    }
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public void setClusterId(long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public List getStores() {
+        return stores;
+    }
+
+    public void setStores(List stores) {
+        this.stores = stores;
+    }
+
+    @Override
+    public Cluster copy() {
+        List stores = null;
+        if (this.stores != null) {
+            stores = Lists.newArrayListWithCapacity(this.stores.size());
+            for (Store store : this.stores) {
+                stores.add(store.copy());
+            }
+        }
+        return new Cluster(this.clusterId, stores);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Cluster cluster = (Cluster) o;
+        return clusterId == cluster.clusterId && Objects.equals(stores, cluster.stores);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(clusterId, stores);
+    }
+
+    @Override
+    public String toString() {
+        return "Cluster{" + "clusterId=" + clusterId + ", stores=" + stores + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Instruction.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Instruction.java
new file mode 100644
index 0000000..a709347
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Instruction.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * Instruction of the placement driver server.
+ *
+ * @author jiachun.fjc
+ */
+public class Instruction implements Serializable {
+
+    private static final long serialVersionUID = 2675841162817080976L;
+
+    private Region            region;
+    private ChangePeer        changePeer;
+    private TransferLeader    transferLeader;
+    private RangeSplit        rangeSplit;
+
+    public Region getRegion() {
+        return region;
+    }
+
+    public void setRegion(Region region) {
+        this.region = region;
+    }
+
+    public ChangePeer getChangePeer() {
+        return changePeer;
+    }
+
+    public void setChangePeer(ChangePeer changePeer) {
+        this.changePeer = changePeer;
+    }
+
+    public TransferLeader getTransferLeader() {
+        return transferLeader;
+    }
+
+    public void setTransferLeader(TransferLeader transferLeader) {
+        this.transferLeader = transferLeader;
+    }
+
+    public RangeSplit getRangeSplit() {
+        return rangeSplit;
+    }
+
+    public void setRangeSplit(RangeSplit rangeSplit) {
+        this.rangeSplit = rangeSplit;
+    }
+
+    @Override
+    public String toString() {
+        return "Instruction{" + "region=" + region + ", changePeer=" + changePeer + ", transferLeader="
+               + transferLeader + ", rangeSplit=" + rangeSplit + '}';
+    }
+
+    public static class ChangePeer implements Serializable {
+
+        private static final long serialVersionUID = -6753587746283650702L;
+
+        // TODO support add/update peer
+    }
+
+    public static class TransferLeader implements Serializable {
+
+        private static final long serialVersionUID = 7483209239871846301L;
+
+        private long              moveToStoreId;
+        private Endpoint          moveToEndpoint;
+
+        public long getMoveToStoreId() {
+            return moveToStoreId;
+        }
+
+        public void setMoveToStoreId(long moveToStoreId) {
+            this.moveToStoreId = moveToStoreId;
+        }
+
+        public Endpoint getMoveToEndpoint() {
+            return moveToEndpoint;
+        }
+
+        public void setMoveToEndpoint(Endpoint moveToEndpoint) {
+            this.moveToEndpoint = moveToEndpoint;
+        }
+
+        @Override
+        public String toString() {
+            return "TransferLeader{" + "moveToStoreId=" + moveToStoreId + ", moveToEndpoint=" + moveToEndpoint + '}';
+        }
+    }
+
+    public static class RangeSplit implements Serializable {
+
+        private static final long serialVersionUID = -3451109819719367744L;
+
+        private Long              newRegionId;
+
+        public Long getNewRegionId() {
+            return newRegionId;
+        }
+
+        public void setNewRegionId(Long newRegionId) {
+            this.newRegionId = newRegionId;
+        }
+
+        @Override
+        public String toString() {
+            return "RangeSplit{" + "newRegionId=" + newRegionId + '}';
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Peer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Peer.java
new file mode 100644
index 0000000..a2b9eb5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Peer.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.alipay.sofa.jraft.util.Copiable;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Peer implements Copiable, Serializable {
+
+    private static final long serialVersionUID = -266370017635677437L;
+
+    private long              id;
+    private long              storeId;
+    private Endpoint          endpoint;
+
+    public Peer() {
+    }
+
+    public Peer(long id, long storeId, Endpoint endpoint) {
+        this.id = id;
+        this.storeId = storeId;
+        this.endpoint = endpoint;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public long getStoreId() {
+        return storeId;
+    }
+
+    public void setStoreId(long storeId) {
+        this.storeId = storeId;
+    }
+
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public void setEndpoint(Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    @Override
+    public Peer copy() {
+        Endpoint endpoint = null;
+        if (this.endpoint != null) {
+            endpoint = this.endpoint.copy();
+        }
+        return new Peer(this.id, this.storeId, endpoint);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Peer peer = (Peer) o;
+        return id == peer.id && storeId == peer.storeId && Objects.equals(endpoint, peer.endpoint);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, storeId, endpoint);
+    }
+
+    @Override
+    public String toString() {
+        return "Peer{" + "id=" + id + ", storeId=" + storeId + ", endpoint=" + endpoint + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/PeerStats.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/PeerStats.java
new file mode 100644
index 0000000..d7a4ea7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/PeerStats.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PeerStats implements Serializable {
+
+    private static final long serialVersionUID = 785959293291029071L;
+
+    private Peer              peer;
+    private int               downSeconds;
+
+    public Peer getPeer() {
+        return peer;
+    }
+
+    public void setPeer(Peer peer) {
+        this.peer = peer;
+    }
+
+    public int getDownSeconds() {
+        return downSeconds;
+    }
+
+    public void setDownSeconds(int downSeconds) {
+        this.downSeconds = downSeconds;
+    }
+
+    @Override
+    public String toString() {
+        return "PeerStats{" + "peer=" + peer + ", downSeconds=" + downSeconds + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Region.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Region.java
new file mode 100644
index 0000000..949d435
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Region.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Copiable;
+
+/**
+ * Region is the most basic kv data unit.  Each region has a left-closed
+ * right-open interval range.  The region is responsible for the crud
+ * request in the range.  Each region is a raft group, which is distributed
+ * on different store nodes. Each region copy is called a region peer.
+ * As the data volume of the region reaches the threshold, it will trigger
+ * split. In fact, it is only the metadata change. This action is very fast.
+ * The two regions just after the split are still on the original store node.
+ * The PD periodically checks the number of regions in each store node, and
+ * the load situation. The raft snapshot is used to migrate the region between
+ * the store nodes to ensure the load balance of the cluster.
+ *
+ * @author jiachun.fjc
+ */
+public class Region implements Copiable, Serializable {
+
+    private static final long serialVersionUID        = -2610978803578899118L;
+
+    // To distinguish the id automatically assigned by the PD,
+    // the manually configured id ranges from [-1, 1000000L).
+    public static final long  MIN_ID_WITH_MANUAL_CONF = -1L;
+    public static final long  MAX_ID_WITH_MANUAL_CONF = 1000000L;
+
+    private long              id;                                             // region id
+    // Region key range [startKey, endKey)
+    private byte[]            startKey;                                       // inclusive
+    private byte[]            endKey;                                         // exclusive
+    private RegionEpoch       regionEpoch;                                    // region term
+    private List        peers;                                          // all peers in the region
+
+    public Region() {
+    }
+
+    public Region(long id, byte[] startKey, byte[] endKey, RegionEpoch regionEpoch, List peers) {
+        this.id = id;
+        this.startKey = startKey;
+        this.endKey = endKey;
+        this.regionEpoch = regionEpoch;
+        this.peers = peers;
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public byte[] getStartKey() {
+        return startKey;
+    }
+
+    public void setStartKey(byte[] startKey) {
+        this.startKey = startKey;
+    }
+
+    public byte[] getEndKey() {
+        return endKey;
+    }
+
+    public void setEndKey(byte[] endKey) {
+        this.endKey = endKey;
+    }
+
+    public RegionEpoch getRegionEpoch() {
+        return regionEpoch;
+    }
+
+    public void setRegionEpoch(RegionEpoch regionEpoch) {
+        this.regionEpoch = regionEpoch;
+    }
+
+    public List getPeers() {
+        return peers;
+    }
+
+    public void setPeers(List peers) {
+        this.peers = peers;
+    }
+
+    @Override
+    public Region copy() {
+        RegionEpoch regionEpoch = null;
+        if (this.regionEpoch != null) {
+            regionEpoch = this.regionEpoch.copy();
+        }
+        List peers = null;
+        if (this.peers != null) {
+            peers = Lists.newArrayListWithCapacity(this.peers.size());
+            for (Peer peer : this.peers) {
+                peers.add(peer.copy());
+            }
+        }
+        return new Region(this.id, this.startKey, this.endKey, regionEpoch, peers);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Region region = (Region) o;
+        return id == region.id && Objects.equals(regionEpoch, region.regionEpoch);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, regionEpoch);
+    }
+
+    @Override
+    public String toString() {
+        return "Region{" + "id=" + id + ", startKey=" + BytesUtil.toHex(startKey) + ", endKey="
+               + BytesUtil.toHex(endKey) + ", regionEpoch=" + regionEpoch + ", peers=" + peers + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionEpoch.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionEpoch.java
new file mode 100644
index 0000000..7a5cee9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionEpoch.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.alipay.sofa.jraft.util.Copiable;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionEpoch implements Copiable, Comparable, Serializable {
+
+    private static final long serialVersionUID = -3752136007698056705L;
+
+    // Conf change version, auto increment when add or remove peer
+    private long              confVer;
+    // Region version, auto increment when split or merge
+    private long              version;
+
+    public RegionEpoch() {
+    }
+
+    public RegionEpoch(long confVer, long version) {
+        this.confVer = confVer;
+        this.version = version;
+    }
+
+    public long getConfVer() {
+        return confVer;
+    }
+
+    public void setConfVer(long confVer) {
+        this.confVer = confVer;
+    }
+
+    public long getVersion() {
+        return version;
+    }
+
+    public void setVersion(long version) {
+        this.version = version;
+    }
+
+    @Override
+    public RegionEpoch copy() {
+        return new RegionEpoch(this.confVer, this.version);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        RegionEpoch that = (RegionEpoch) o;
+        return confVer == that.confVer && version == that.version;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(confVer, version);
+    }
+
+    @Override
+    public String toString() {
+        return "RegionEpoch{" + "confVer=" + confVer + ", version=" + version + '}';
+    }
+
+    @Override
+    public int compareTo(RegionEpoch o) {
+        if (this.version == o.version) {
+            return (int) (this.confVer - o.confVer);
+        }
+        return (int) (this.version - o.version);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionStats.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionStats.java
new file mode 100644
index 0000000..3cf361c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/RegionStats.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionStats implements Serializable {
+
+    private static final long serialVersionUID = -4790131346095689335L;
+
+    private long              regionId;
+    // Leader Peer sending the heartbeat
+    private Peer              leader;
+    // Leader considers that these peers are down
+    private List   downPeers;
+    // Pending peers are the peers that the leader can't consider as working followers
+    private List   pendingPeers;
+    // Bytes written for the region during this period
+    private long              bytesWritten;
+    // Bytes read for the region during this period
+    private long              bytesRead;
+    // Keys written for the region during this period
+    private long              keysWritten;
+    // Keys read for the region during this period
+    private long              keysRead;
+    // Approximate region size
+    private long              approximateSize;
+    // Approximate number of keys
+    private long              approximateKeys;
+    // Actually reported time interval
+    private TimeInterval      interval;
+
+    public long getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(long regionId) {
+        this.regionId = regionId;
+    }
+
+    public Peer getLeader() {
+        return leader;
+    }
+
+    public void setLeader(Peer leader) {
+        this.leader = leader;
+    }
+
+    public List getDownPeers() {
+        return downPeers;
+    }
+
+    public void setDownPeers(List downPeers) {
+        this.downPeers = downPeers;
+    }
+
+    public List getPendingPeers() {
+        return pendingPeers;
+    }
+
+    public void setPendingPeers(List pendingPeers) {
+        this.pendingPeers = pendingPeers;
+    }
+
+    public long getBytesWritten() {
+        return bytesWritten;
+    }
+
+    public void setBytesWritten(long bytesWritten) {
+        this.bytesWritten = bytesWritten;
+    }
+
+    public long getBytesRead() {
+        return bytesRead;
+    }
+
+    public void setBytesRead(long bytesRead) {
+        this.bytesRead = bytesRead;
+    }
+
+    public long getKeysWritten() {
+        return keysWritten;
+    }
+
+    public void setKeysWritten(long keysWritten) {
+        this.keysWritten = keysWritten;
+    }
+
+    public long getKeysRead() {
+        return keysRead;
+    }
+
+    public void setKeysRead(long keysRead) {
+        this.keysRead = keysRead;
+    }
+
+    public long getApproximateSize() {
+        return approximateSize;
+    }
+
+    public void setApproximateSize(long approximateSize) {
+        this.approximateSize = approximateSize;
+    }
+
+    public long getApproximateKeys() {
+        return approximateKeys;
+    }
+
+    public void setApproximateKeys(long approximateKeys) {
+        this.approximateKeys = approximateKeys;
+    }
+
+    public TimeInterval getInterval() {
+        return interval;
+    }
+
+    public void setInterval(TimeInterval interval) {
+        this.interval = interval;
+    }
+
+    @Override
+    public String toString() {
+        return "RegionStats{" + "regionId=" + regionId + ", leader=" + leader + ", downPeers=" + downPeers
+               + ", pendingPeers=" + pendingPeers + ", bytesWritten=" + bytesWritten + ", bytesRead=" + bytesRead
+               + ", keysWritten=" + keysWritten + ", keysRead=" + keysRead + ", approximateSize=" + approximateSize
+               + ", approximateKeys=" + approximateKeys + ", interval=" + interval + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Store.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Store.java
new file mode 100644
index 0000000..1eb6f39
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/Store.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Objects;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.Copiable;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * A physical node in the cluster, embedded in an app java process,
+ * store corresponds to a local file based rocksDB, a store contains
+ * multiple regions, all regions share a rocksDB.
+ *
+ * @author jiachun.fjc
+ */
+public class Store implements Copiable, Serializable {
+
+    private static final long serialVersionUID = 8566110829366373797L;
+
+    private long              id;                                     // store id
+    private Endpoint          endpoint;                               // address
+    private StoreState        state;                                  // store's state
+    private List      regions;                                // list of included regions
+    private List  labels;                                 // key/value label
+
+    public Store() {
+    }
+
+    public Store(long id, Endpoint endpoint, StoreState state, List regions, List labels) {
+        this.id = id;
+        this.endpoint = endpoint;
+        this.state = state;
+        this.regions = regions;
+        this.labels = labels;
+    }
+
+    public boolean isEmpty() {
+        return this.endpoint == null || this.regions == null || this.regions.isEmpty();
+    }
+
+    public long getId() {
+        return id;
+    }
+
+    public void setId(long id) {
+        this.id = id;
+    }
+
+    public Endpoint getEndpoint() {
+        return endpoint;
+    }
+
+    public void setEndpoint(Endpoint endpoint) {
+        this.endpoint = endpoint;
+    }
+
+    public List getRegions() {
+        return regions;
+    }
+
+    public void setRegions(List regions) {
+        this.regions = regions;
+    }
+
+    @Override
+    public Store copy() {
+        Endpoint endpoint = null;
+        if (this.endpoint != null) {
+            endpoint = this.endpoint.copy();
+        }
+        List regions = null;
+        if (this.regions != null) {
+            regions = Lists.newArrayListWithCapacity(this.regions.size());
+            for (Region region : this.regions) {
+                regions.add(region.copy());
+            }
+        }
+        List labels = null;
+        if (this.labels != null) {
+            labels = Lists.newArrayListWithCapacity(this.labels.size());
+            for (StoreLabel label : this.labels) {
+                labels.add(label.copy());
+            }
+        }
+        return new Store(this.id, endpoint, this.state, regions, labels);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Store store = (Store) o;
+        return id == store.id && Objects.equals(endpoint, store.endpoint);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id, endpoint);
+    }
+
+    @Override
+    public String toString() {
+        return "Store{" + "id=" + id + ", endpoint=" + endpoint + ", state=" + state + ", regions=" + regions
+               + ", labels=" + labels + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreLabel.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreLabel.java
new file mode 100644
index 0000000..b2384e8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreLabel.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.util.Copiable;
+
+/**
+ * Case insensitive key/value for replica constraints.
+ *
+ * @author jiachun.fjc
+ */
+public class StoreLabel implements Copiable, Serializable {
+
+    private static final long serialVersionUID = 5672444723199723795L;
+
+    private String            key;
+    private String            value;
+
+    public StoreLabel() {
+    }
+
+    public StoreLabel(String key, String value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    @Override
+    public StoreLabel copy() {
+        return new StoreLabel(this.key, this.value);
+    }
+
+    @Override
+    public String toString() {
+        return "StoreLabel{" + "key='" + key + '\'' + ", value='" + value + '\'' + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreState.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreState.java
new file mode 100644
index 0000000..8ba8ba7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreState.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public enum StoreState {
+    UP, OFFLINE, TOMBSTONE
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreStats.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreStats.java
new file mode 100644
index 0000000..0290784
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/StoreStats.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StoreStats implements Serializable {
+
+    private static final long serialVersionUID = -7818958068467754379L;
+
+    private long              storeId;
+    // Capacity for the store
+    private long              capacity;
+    // Available size for the store
+    private long              available;
+    // Total region count in this store
+    private int               regionCount;
+    // Leader region count in this store
+    private int               leaderRegionCount;
+    // Current sending snapshot count
+    private int               sendingSnapCount;
+    // Current receiving snapshot count
+    private int               receivingSnapCount;
+    // How many region is applying snapshot
+    private int               applyingSnapCount;
+    // When the store is started (unix timestamp in milliseconds)
+    private long              startTime;
+    // If the store is busy
+    private boolean           busy;
+    // Actually used space by db
+    private long              usedSize;
+    // Bytes written for the store during this period
+    private long              bytesWritten;
+    // Bytes read for the store during this period
+    private long              bytesRead;
+    // Keys written for the store during this period
+    private long              keysWritten;
+    // Keys read for the store during this period
+    private long              keysRead;
+    // Actually reported time interval
+    private TimeInterval      interval;
+
+    public long getStoreId() {
+        return storeId;
+    }
+
+    public void setStoreId(long storeId) {
+        this.storeId = storeId;
+    }
+
+    public long getCapacity() {
+        return capacity;
+    }
+
+    public void setCapacity(long capacity) {
+        this.capacity = capacity;
+    }
+
+    public long getAvailable() {
+        return available;
+    }
+
+    public void setAvailable(long available) {
+        this.available = available;
+    }
+
+    public int getRegionCount() {
+        return regionCount;
+    }
+
+    public void setRegionCount(int regionCount) {
+        this.regionCount = regionCount;
+    }
+
+    public int getLeaderRegionCount() {
+        return leaderRegionCount;
+    }
+
+    public void setLeaderRegionCount(int leaderRegionCount) {
+        this.leaderRegionCount = leaderRegionCount;
+    }
+
+    public int getSendingSnapCount() {
+        return sendingSnapCount;
+    }
+
+    public void setSendingSnapCount(int sendingSnapCount) {
+        this.sendingSnapCount = sendingSnapCount;
+    }
+
+    public int getReceivingSnapCount() {
+        return receivingSnapCount;
+    }
+
+    public void setReceivingSnapCount(int receivingSnapCount) {
+        this.receivingSnapCount = receivingSnapCount;
+    }
+
+    public int getApplyingSnapCount() {
+        return applyingSnapCount;
+    }
+
+    public void setApplyingSnapCount(int applyingSnapCount) {
+        this.applyingSnapCount = applyingSnapCount;
+    }
+
+    public long getStartTime() {
+        return startTime;
+    }
+
+    public void setStartTime(long startTime) {
+        this.startTime = startTime;
+    }
+
+    public boolean isBusy() {
+        return busy;
+    }
+
+    public void setBusy(boolean busy) {
+        this.busy = busy;
+    }
+
+    public long getUsedSize() {
+        return usedSize;
+    }
+
+    public void setUsedSize(long usedSize) {
+        this.usedSize = usedSize;
+    }
+
+    public long getBytesWritten() {
+        return bytesWritten;
+    }
+
+    public void setBytesWritten(long bytesWritten) {
+        this.bytesWritten = bytesWritten;
+    }
+
+    public long getBytesRead() {
+        return bytesRead;
+    }
+
+    public void setBytesRead(long bytesRead) {
+        this.bytesRead = bytesRead;
+    }
+
+    public long getKeysWritten() {
+        return keysWritten;
+    }
+
+    public void setKeysWritten(long keysWritten) {
+        this.keysWritten = keysWritten;
+    }
+
+    public long getKeysRead() {
+        return keysRead;
+    }
+
+    public void setKeysRead(long keysRead) {
+        this.keysRead = keysRead;
+    }
+
+    public TimeInterval getInterval() {
+        return interval;
+    }
+
+    public void setInterval(TimeInterval interval) {
+        this.interval = interval;
+    }
+
+    @Override
+    public String toString() {
+        return "StoreStats{" + "storeId=" + storeId + ", capacity=" + capacity + ", available=" + available
+               + ", regionCount=" + regionCount + ", leaderRegionCount=" + leaderRegionCount + ", sendingSnapCount="
+               + sendingSnapCount + ", receivingSnapCount=" + receivingSnapCount + ", applyingSnapCount="
+               + applyingSnapCount + ", startTime=" + startTime + ", busy=" + busy + ", usedSize=" + usedSize
+               + ", bytesWritten=" + bytesWritten + ", bytesRead=" + bytesRead + ", keysWritten=" + keysWritten
+               + ", keysRead=" + keysRead + ", interval=" + interval + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/TimeInterval.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/TimeInterval.java
new file mode 100644
index 0000000..5c0feee
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metadata/TimeInterval.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metadata;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.util.Copiable;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class TimeInterval implements Copiable, Serializable {
+
+    private static final long serialVersionUID = 2454987958697466235L;
+
+    // The unix timestamp in milliseconds of the start of this period.
+    private long              startTimestamp;
+    // The unix timestamp in seconds of the end of this period.
+    private long              endTimestamp;
+
+    public TimeInterval() {
+    }
+
+    public TimeInterval(long startTimestamp, long endTimestamp) {
+        this.startTimestamp = startTimestamp;
+        this.endTimestamp = endTimestamp;
+    }
+
+    public long getStartTimestamp() {
+        return startTimestamp;
+    }
+
+    public void setStartTimestamp(long startTimestamp) {
+        this.startTimestamp = startTimestamp;
+    }
+
+    public long getEndTimestamp() {
+        return endTimestamp;
+    }
+
+    public void setEndTimestamp(long endTimestamp) {
+        this.endTimestamp = endTimestamp;
+    }
+
+    @Override
+    public TimeInterval copy() {
+        return new TimeInterval(this.startTimestamp, this.endTimestamp);
+    }
+
+    @Override
+    public String toString() {
+        return "TimeInterval{" + "startTimestamp=" + startTimestamp + ", endTimestamp=" + endTimestamp + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetricNames.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetricNames.java
new file mode 100644
index 0000000..3a27519
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetricNames.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metrics;
+
+/**
+ * KVMetrics prefix names.
+ *
+ * @author jiachun.fjc
+ */
+public class KVMetricNames {
+
+    // for state machine
+    public static final String STATE_MACHINE_APPLY_QPS   = "rhea-st-apply-qps";
+    public static final String STATE_MACHINE_BATCH_WRITE = "rhea-st-batch-write";
+
+    // for rpc
+    public static final String RPC_REQUEST_HANDLE_TIMER  = "rhea-rpc-request-timer";
+
+    public static final String DB_TIMER                  = "rhea-db-timer";
+
+    public static final String REGION_KEYS_READ          = "rhea-region-keys-read";
+    public static final String REGION_KEYS_WRITTEN       = "rhea-region-keys-written";
+
+    public static final String REGION_BYTES_READ         = "rhea-region-bytes-read";
+    public static final String REGION_BYTES_WRITTEN      = "rhea-region-bytes-written";
+
+    public static final String SEND_BATCHING             = "send_batching";
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetrics.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetrics.java
new file mode 100644
index 0000000..0622efe
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/metrics/KVMetrics.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.metrics;
+
+import com.alipay.sofa.jraft.rhea.util.StringBuilderHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Timer;
+
+/**
+ * In rheaKV, metrics are required. As for whether to output (log) metrics results, you decide.
+ *
+ * @author jiachun.fjc
+ */
+public final class KVMetrics {
+
+    private static final MetricRegistry metricRegistry = new MetricRegistry();
+
+    /**
+     * Return the global registry of metric instances.
+     */
+    public static MetricRegistry metricRegistry() {
+        return metricRegistry;
+    }
+
+    /**
+     * Return the {@link Meter} registered under this name; or create
+     * and register a new {@link Meter} if none is registered.
+     */
+    public static Meter meter(final String name) {
+        return metricRegistry.meter(Requires.requireNonNull(name, "name"));
+    }
+
+    /**
+     * Return the {@link Meter} registered under this name; or create
+     * and register a new {@link Meter} if none is registered.
+     */
+    public static Meter meter(final String... names) {
+        return metricRegistry.meter(name(names));
+    }
+
+    /**
+     * Return the {@link Timer} registered under this name; or create
+     * and register a new {@link Timer} if none is registered.
+     */
+    public static Timer timer(final String name) {
+        return metricRegistry.timer(Requires.requireNonNull(name, "name"));
+    }
+
+    /**
+     * Return the {@link Timer} registered under this name; or create
+     * and register a new {@link Timer} if none is registered.
+     */
+    public static Timer timer(final String... names) {
+        return metricRegistry.timer(name(names));
+    }
+
+    /**
+     * Return the {@link Counter} registered under this name; or create
+     * and register a new {@link Counter} if none is registered.
+     */
+    public static Counter counter(final String name) {
+        return metricRegistry.counter(Requires.requireNonNull(name, "name"));
+    }
+
+    /**
+     * Return the {@link Counter} registered under this name; or create
+     * and register a new {@link Counter} if none is registered.
+     */
+    public static Counter counter(final String... names) {
+        return metricRegistry.counter(name(names));
+    }
+
+    /**
+     * Return the {@link Histogram} registered under this name; or create
+     * and register a new {@link Histogram} if none is registered.
+     */
+    public static Histogram histogram(final String name) {
+        return metricRegistry.histogram(Requires.requireNonNull(name, "name"));
+    }
+
+    /**
+     * Return the {@link Histogram} registered under this name; or create
+     * and register a new {@link Histogram} if none is registered.
+     */
+    public static Histogram histogram(final String... names) {
+        return metricRegistry.histogram(name(names));
+    }
+
+    private static String name(final String... names) {
+        final StringBuilder buf = StringBuilderHelper.get();
+        for (final String name : names) {
+            if (buf.length() > 0) {
+                buf.append('_');
+            }
+            buf.append(name);
+        }
+        return buf.toString();
+    }
+
+    private KVMetrics() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/BatchingOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/BatchingOptions.java
new file mode 100644
index 0000000..31f819a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/BatchingOptions.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BatchingOptions {
+
+    // If batching is allowed, the client will submit the requests in batch mode,
+    // which will improve the throughput without any negative impact on the delay.
+    private boolean allowBatching = true;
+    // Maximum number of requests that can be applied in a batch.
+    private int     batchSize     = 100;
+    // Internal disruptor buffers size for get/put request etc.
+    private int     bufSize       = 8192;
+    // Maximum bytes size to cached for put-request (keys.size + value.size).
+    private int     maxWriteBytes = 32768;
+    // Maximum bytes size to cached for get-request (keys.size).
+    private int     maxReadBytes  = 1024;
+
+    public boolean isAllowBatching() {
+        return allowBatching;
+    }
+
+    public void setAllowBatching(boolean allowBatching) {
+        this.allowBatching = allowBatching;
+    }
+
+    public int getBatchSize() {
+        return batchSize;
+    }
+
+    public void setBatchSize(int batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public int getBufSize() {
+        return bufSize;
+    }
+
+    public void setBufSize(int bufSize) {
+        this.bufSize = bufSize;
+    }
+
+    public int getMaxWriteBytes() {
+        return maxWriteBytes;
+    }
+
+    public void setMaxWriteBytes(int maxWriteBytes) {
+        this.maxWriteBytes = maxWriteBytes;
+    }
+
+    public int getMaxReadBytes() {
+        return maxReadBytes;
+    }
+
+    public void setMaxReadBytes(int maxReadBytes) {
+        this.maxReadBytes = maxReadBytes;
+    }
+
+    @Override
+    public String toString() {
+        return "BatchingOptions{" + "allowBatching=" + allowBatching + ", batchSize=" + batchSize + ", bufSize="
+               + bufSize + ", maxWriteBytes=" + maxWriteBytes + ", maxReadBytes=" + maxReadBytes + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/HeartbeatOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/HeartbeatOptions.java
new file mode 100644
index 0000000..fcc4d3c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/HeartbeatOptions.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+/**
+ * @author jiachun.fjc
+ */
+public class HeartbeatOptions {
+
+    private long storeHeartbeatIntervalSeconds  = 60;
+    private long regionHeartbeatIntervalSeconds = 60;
+    private int  heartbeatRpcTimeoutMillis      = 5000;
+
+    public long getStoreHeartbeatIntervalSeconds() {
+        return storeHeartbeatIntervalSeconds;
+    }
+
+    public void setStoreHeartbeatIntervalSeconds(long storeHeartbeatIntervalSeconds) {
+        this.storeHeartbeatIntervalSeconds = storeHeartbeatIntervalSeconds;
+    }
+
+    public long getRegionHeartbeatIntervalSeconds() {
+        return regionHeartbeatIntervalSeconds;
+    }
+
+    public void setRegionHeartbeatIntervalSeconds(long regionHeartbeatIntervalSeconds) {
+        this.regionHeartbeatIntervalSeconds = regionHeartbeatIntervalSeconds;
+    }
+
+    public int getHeartbeatRpcTimeoutMillis() {
+        return heartbeatRpcTimeoutMillis;
+    }
+
+    public void setHeartbeatRpcTimeoutMillis(int heartbeatRpcTimeoutMillis) {
+        this.heartbeatRpcTimeoutMillis = heartbeatRpcTimeoutMillis;
+    }
+
+    @Override
+    public String toString() {
+        return "HeartbeatOptions{" + ", storeHeartbeatIntervalSeconds=" + storeHeartbeatIntervalSeconds
+               + ", regionHeartbeatIntervalSeconds=" + regionHeartbeatIntervalSeconds + ", heartbeatRpcTimeoutMillis="
+               + heartbeatRpcTimeoutMillis + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/MemoryDBOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/MemoryDBOptions.java
new file mode 100644
index 0000000..f59e907
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/MemoryDBOptions.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MemoryDBOptions {
+
+    // for segment snapshot file size
+    private int keysPerSegment = 4096;
+
+    public int getKeysPerSegment() {
+        return keysPerSegment;
+    }
+
+    public void setKeysPerSegment(int keysPerSegment) {
+        this.keysPerSegment = keysPerSegment;
+    }
+
+    @Override
+    public String toString() {
+        return "MemoryDBOptions{" + "keysPerSegment=" + keysPerSegment + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverOptions.java
new file mode 100644
index 0000000..999222a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverOptions.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.option.CliOptions;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class PlacementDriverOptions {
+
+    private boolean                       fake;
+    private CliOptions                    cliOptions;
+    private RpcOptions                    pdRpcOptions;
+    // placement driver raft group id
+    private String                        pdGroupId;
+    private List regionRouteTableOptionsList;
+    private String                        initialServerList;
+    // placement driver server address list, with ',' as a separator
+    private String                        initialPdServerList;
+
+    public boolean isFake() {
+        return fake;
+    }
+
+    public void setFake(boolean fake) {
+        this.fake = fake;
+    }
+
+    public CliOptions getCliOptions() {
+        return cliOptions;
+    }
+
+    public void setCliOptions(CliOptions cliOptions) {
+        this.cliOptions = cliOptions;
+    }
+
+    public RpcOptions getPdRpcOptions() {
+        return pdRpcOptions;
+    }
+
+    public void setPdRpcOptions(RpcOptions pdRpcOptions) {
+        this.pdRpcOptions = pdRpcOptions;
+    }
+
+    public String getPdGroupId() {
+        return pdGroupId;
+    }
+
+    public void setPdGroupId(String pdGroupId) {
+        this.pdGroupId = pdGroupId;
+    }
+
+    public List getRegionRouteTableOptionsList() {
+        return regionRouteTableOptionsList;
+    }
+
+    public void setRegionRouteTableOptionsList(List regionRouteTableOptionsList) {
+        this.regionRouteTableOptionsList = regionRouteTableOptionsList;
+    }
+
+    public String getInitialServerList() {
+        return initialServerList;
+    }
+
+    public void setInitialServerList(String initialServerList) {
+        this.initialServerList = initialServerList;
+    }
+
+    public String getInitialPdServerList() {
+        return initialPdServerList;
+    }
+
+    public void setInitialPdServerList(String initialPdServerList) {
+        this.initialPdServerList = initialPdServerList;
+    }
+
+    @Override
+    public String toString() {
+        return "PlacementDriverOptions{" + "fake=" + fake + ", cliOptions=" + cliOptions + ", pdRpcOptions="
+               + pdRpcOptions + ", pdGroupId='" + pdGroupId + '\'' + ", regionRouteTableOptionsList="
+               + regionRouteTableOptionsList + ", initialServerList='" + initialServerList + '\''
+               + ", initialPdServerList='" + initialPdServerList + '\'' + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionEngineOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionEngineOptions.java
new file mode 100644
index 0000000..40b7d03
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionEngineOptions.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Copiable;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionEngineOptions implements Copiable {
+
+    private Long        regionId;
+    private String      startKey;
+    private byte[]      startKeyBytes;
+    private String      endKey;
+    private byte[]      endKeyBytes;
+    private NodeOptions nodeOptions;
+    // Should extends from StoreEngineOptions
+    private String      raftGroupId;
+    // Should extends from StoreEngineOptions
+    private String      raftDataPath;
+    // Should extends from StoreEngineOptions
+    private Endpoint    serverAddress;
+    // Should extends from StoreEngineOptions
+    private String      initialServerList;
+    // Can extends from StoreEngineOptions
+    private long        metricsReportPeriod;
+
+    public Long getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(Long regionId) {
+        this.regionId = regionId;
+    }
+
+    public String getStartKey() {
+        return startKey;
+    }
+
+    public void setStartKey(String startKey) {
+        this.startKey = startKey;
+        this.startKeyBytes = BytesUtil.writeUtf8(startKey);
+    }
+
+    public byte[] getStartKeyBytes() {
+        return startKeyBytes;
+    }
+
+    public void setStartKeyBytes(byte[] startKeyBytes) {
+        this.startKeyBytes = startKeyBytes;
+    }
+
+    public String getEndKey() {
+        return endKey;
+    }
+
+    public void setEndKey(String endKey) {
+        this.endKey = endKey;
+        this.endKeyBytes = BytesUtil.writeUtf8(endKey);
+    }
+
+    public byte[] getEndKeyBytes() {
+        return endKeyBytes;
+    }
+
+    public void setEndKeyBytes(byte[] endKeyBytes) {
+        this.endKeyBytes = endKeyBytes;
+    }
+
+    public NodeOptions getNodeOptions() {
+        return nodeOptions;
+    }
+
+    public void setNodeOptions(NodeOptions nodeOptions) {
+        this.nodeOptions = nodeOptions;
+    }
+
+    public String getRaftGroupId() {
+        return raftGroupId;
+    }
+
+    public void setRaftGroupId(String raftGroupId) {
+        this.raftGroupId = raftGroupId;
+    }
+
+    public String getRaftDataPath() {
+        return raftDataPath;
+    }
+
+    public void setRaftDataPath(String raftDataPath) {
+        this.raftDataPath = raftDataPath;
+    }
+
+    public Endpoint getServerAddress() {
+        return serverAddress;
+    }
+
+    public void setServerAddress(Endpoint serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    public String getInitialServerList() {
+        return initialServerList;
+    }
+
+    public void setInitialServerList(String initialServerList) {
+        this.initialServerList = initialServerList;
+    }
+
+    public long getMetricsReportPeriod() {
+        return metricsReportPeriod;
+    }
+
+    public void setMetricsReportPeriod(long metricsReportPeriod) {
+        this.metricsReportPeriod = metricsReportPeriod;
+    }
+
+    @Override
+    public RegionEngineOptions copy() {
+        final RegionEngineOptions copy = new RegionEngineOptions();
+        copy.setRegionId(this.regionId);
+        copy.setStartKey(this.startKey);
+        copy.setStartKeyBytes(this.startKeyBytes);
+        copy.setEndKey(this.endKey);
+        copy.setEndKeyBytes(this.endKeyBytes);
+        copy.setNodeOptions(this.nodeOptions == null ? new NodeOptions() : this.nodeOptions.copy());
+        copy.setRaftGroupId(this.raftGroupId);
+        copy.setRaftDataPath(this.raftDataPath);
+        copy.setServerAddress(this.serverAddress);
+        copy.setInitialServerList(this.initialServerList);
+        copy.setMetricsReportPeriod(this.metricsReportPeriod);
+        return copy;
+    }
+
+    @Override
+    public String toString() {
+        return "RegionEngineOptions{" + "regionId=" + regionId + ", startKey='" + startKey + '\'' + ", startKeyBytes="
+               + BytesUtil.toHex(startKeyBytes) + ", endKey='" + endKey + '\'' + ", endKeyBytes="
+               + BytesUtil.toHex(endKeyBytes) + ", raftGroupId='" + raftGroupId + '\'' + ", raftDataPath='"
+               + raftDataPath + '\'' + ", nodeOptions=" + nodeOptions + ", serverAddress=" + serverAddress
+               + ", initialServerList='" + initialServerList + '\'' + ", metricsReportPeriod=" + metricsReportPeriod
+               + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionRouteTableOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionRouteTableOptions.java
new file mode 100644
index 0000000..3fbe538
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RegionRouteTableOptions.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RegionRouteTableOptions {
+
+    private Long   regionId;
+    private String startKey;
+    private byte[] startKeyBytes;
+    private String endKey;
+    private byte[] endKeyBytes;
+    // Can extends from RheaKVStoreOptions
+    private String initialServerList;
+
+    public Long getRegionId() {
+        return regionId;
+    }
+
+    public void setRegionId(Long regionId) {
+        this.regionId = regionId;
+    }
+
+    public String getStartKey() {
+        return startKey;
+    }
+
+    public void setStartKey(String startKey) {
+        this.startKey = startKey;
+        this.startKeyBytes = BytesUtil.writeUtf8(startKey);
+    }
+
+    public byte[] getStartKeyBytes() {
+        return startKeyBytes;
+    }
+
+    public void setStartKeyBytes(byte[] startKeyBytes) {
+        this.startKeyBytes = startKeyBytes;
+    }
+
+    public String getEndKey() {
+        return endKey;
+    }
+
+    public void setEndKey(String endKey) {
+        this.endKey = endKey;
+        this.endKeyBytes = BytesUtil.writeUtf8(endKey);
+    }
+
+    public byte[] getEndKeyBytes() {
+        return endKeyBytes;
+    }
+
+    public void setEndKeyBytes(byte[] endKeyBytes) {
+        this.endKeyBytes = endKeyBytes;
+    }
+
+    public String getInitialServerList() {
+        return initialServerList;
+    }
+
+    public void setInitialServerList(String initialServerList) {
+        this.initialServerList = initialServerList;
+    }
+
+    @Override
+    public String toString() {
+        return "RegionRouteTableOptions{" + "regionId=" + regionId + ", startKey='" + startKey + '\''
+               + ", startKeyBytes=" + BytesUtil.toHex(startKeyBytes) + ", endKey='" + endKey + '\'' + ", endKeyBytes="
+               + BytesUtil.toHex(endKeyBytes) + ", initialServerList='" + initialServerList + '\'' + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RheaKVStoreOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RheaKVStoreOptions.java
new file mode 100644
index 0000000..66263b3
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RheaKVStoreOptions.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import com.alipay.sofa.jraft.rhea.options.configured.BatchingOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RpcOptionsConfigured;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVStoreOptions {
+
+    // A clusterId is required to connect to the PD server, and PD server
+    // use clusterId isolate different cluster.  The fake PD mode does not
+    // need to be configured.
+    private long                   clusterId;
+    // Each store node contains one or more raft-group replication groups.
+    // This field is the name prefix of all replication groups. All raft-group
+    // names follow the naming rules of [clusterName-regionId].
+    private String                 clusterName           = "default-group-cluster";
+    private PlacementDriverOptions placementDriverOptions;
+    private StoreEngineOptions     storeEngineOptions;
+    // Initial server node list.
+    private String                 initialServerList;
+    // Whether to read data only from the leader node, reading from the
+    // follower node can also ensure consistent reading, but the reading
+    // delay may increase due to the delay of the follower synchronization
+    // data.
+    private boolean                onlyLeaderRead        = true;
+    private RpcOptions             rpcOptions            = RpcOptionsConfigured.newDefaultConfig();
+    private int                    failoverRetries       = 2;
+    private long                   futureTimeoutMillis   = 5000;
+    private boolean                useParallelKVExecutor = true;
+    private BatchingOptions        batchingOptions       = BatchingOptionsConfigured.newDefaultConfig();
+    // If 'useParallelCompress' is true , We will compress and decompress Snapshot concurrently
+    private boolean                useParallelCompress   = false;
+    private int                    compressThreads       = Utils.cpus();
+    private int                    deCompressThreads     = Utils.cpus() + 1;
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public void setClusterId(long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public void setClusterName(String clusterName) {
+        this.clusterName = clusterName;
+    }
+
+    /**
+     * Same as {@link #getClusterName()}
+     */
+    public String getMultiRaftGroupClusterName() {
+        return clusterName;
+    }
+
+    /**
+     * Same as {@link #setClusterName(String)}
+     */
+    public void setMultiRaftGroupClusterName(String clusterName) {
+        this.clusterName = clusterName;
+    }
+
+    public PlacementDriverOptions getPlacementDriverOptions() {
+        return placementDriverOptions;
+    }
+
+    public void setPlacementDriverOptions(PlacementDriverOptions placementDriverOptions) {
+        this.placementDriverOptions = placementDriverOptions;
+    }
+
+    public StoreEngineOptions getStoreEngineOptions() {
+        return storeEngineOptions;
+    }
+
+    public void setStoreEngineOptions(StoreEngineOptions storeEngineOptions) {
+        this.storeEngineOptions = storeEngineOptions;
+    }
+
+    public String getInitialServerList() {
+        return initialServerList;
+    }
+
+    public void setInitialServerList(String initialServerList) {
+        this.initialServerList = initialServerList;
+    }
+
+    public RpcOptions getRpcOptions() {
+        return rpcOptions;
+    }
+
+    public void setRpcOptions(RpcOptions rpcOptions) {
+        this.rpcOptions = rpcOptions;
+    }
+
+    public boolean isOnlyLeaderRead() {
+        return onlyLeaderRead;
+    }
+
+    public void setOnlyLeaderRead(boolean onlyLeaderRead) {
+        this.onlyLeaderRead = onlyLeaderRead;
+    }
+
+    public int getFailoverRetries() {
+        return failoverRetries;
+    }
+
+    public void setFailoverRetries(int failoverRetries) {
+        this.failoverRetries = failoverRetries;
+    }
+
+    public long getFutureTimeoutMillis() {
+        return futureTimeoutMillis;
+    }
+
+    public void setFutureTimeoutMillis(long futureTimeoutMillis) {
+        this.futureTimeoutMillis = futureTimeoutMillis;
+    }
+
+    public boolean isUseParallelKVExecutor() {
+        return useParallelKVExecutor;
+    }
+
+    public void setUseParallelKVExecutor(boolean useParallelKVExecutor) {
+        this.useParallelKVExecutor = useParallelKVExecutor;
+    }
+
+    public BatchingOptions getBatchingOptions() {
+        return batchingOptions;
+    }
+
+    public void setBatchingOptions(BatchingOptions batchingOptions) {
+        this.batchingOptions = batchingOptions;
+    }
+
+    public boolean isUseParallelCompress() {
+        return useParallelCompress;
+    }
+
+    public void setUseParallelCompress(boolean useParallelCompress) {
+        this.useParallelCompress = useParallelCompress;
+    }
+
+    public int getCompressThreads() {
+        return compressThreads;
+    }
+
+    public void setCompressThreads(int compressThreads) {
+        this.compressThreads = compressThreads;
+    }
+
+    public int getDeCompressThreads() {
+        return deCompressThreads;
+    }
+
+    public void setDeCompressThreads(int deCompressThreads) {
+        this.deCompressThreads = deCompressThreads;
+    }
+
+    @Override
+    public String toString() {
+        return "RheaKVStoreOptions{" + "clusterId=" + clusterId + ", clusterName='" + clusterName + '\''
+               + ", placementDriverOptions=" + placementDriverOptions + ", storeEngineOptions=" + storeEngineOptions
+               + ", initialServerList='" + initialServerList + '\'' + ", onlyLeaderRead=" + onlyLeaderRead
+               + ", rpcOptions=" + rpcOptions + ", failoverRetries=" + failoverRetries + ", futureTimeoutMillis="
+               + futureTimeoutMillis + ", useParallelKVExecutor=" + useParallelKVExecutor + ", batchingOptions="
+               + batchingOptions + ", useParallelCompress=" + useParallelCompress + ", compressThreads="
+               + compressThreads + ", deCompressThreads=" + deCompressThreads + '}';
+    }
+
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RocksDBOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RocksDBOptions.java
new file mode 100644
index 0000000..64e8910
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RocksDBOptions.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+/**
+ *
+ * @author dennis
+ * @author jiachun.fjc
+ */
+public class RocksDBOptions {
+
+    // The raft log used fsync by default, and the correctness of
+    // state-machine data with rheakv depends on the raft log + snapshot,
+    // so we do not need to fsync.
+    private boolean sync                              = false;
+    // For the same reason(See the comment of ‘sync’ field), we also
+    // don't need WAL, which can improve performance.
+    //
+    // If `sync` is true, `disableWAL` must be set false
+    private boolean disableWAL                        = true;
+    // https://github.com/facebook/rocksdb/wiki/Checkpoints
+    private boolean fastSnapshot                      = false;
+    private boolean asyncSnapshot                     = false;
+    // Statistics to analyze the performance of db
+    private boolean openStatisticsCollector           = true;
+    private long    statisticsCallbackIntervalSeconds = 0;
+    private String  dbPath;
+
+    public boolean isSync() {
+        return sync;
+    }
+
+    /**
+     * If true, the write will be flushed from the operating system
+     * buffer cache (by calling WritableFile::Sync()) before the write
+     * is considered complete.  If this flag is true, writes will be
+     * slower.
+     *
+     * If this flag is false, and the machine crashes, some recent
+     * writes may be lost.  Note that if it is just the process that
+     * crashes (i.e., the machine does not reboot), no writes will be
+     * lost even if sync==false.
+     *
+     * In other words, a DB write with sync==false has similar
+     * crash semantics as the "write()" system call.  A DB write
+     * with sync==true has similar crash semantics to a "write()"
+     * system call followed by "fdatasync()".
+     */
+    public void setSync(boolean sync) {
+        this.sync = sync;
+    }
+
+    public boolean isDisableWAL() {
+        return disableWAL;
+    }
+
+    public void setDisableWAL(boolean disableWAL) {
+        this.disableWAL = disableWAL;
+    }
+
+    public boolean isFastSnapshot() {
+        return fastSnapshot;
+    }
+
+    public void setFastSnapshot(boolean fastSnapshot) {
+        this.fastSnapshot = fastSnapshot;
+    }
+
+    public boolean isAsyncSnapshot() {
+        return asyncSnapshot;
+    }
+
+    public void setAsyncSnapshot(boolean asyncSnapshot) {
+        this.asyncSnapshot = asyncSnapshot;
+    }
+
+    public boolean isOpenStatisticsCollector() {
+        return openStatisticsCollector;
+    }
+
+    public void setOpenStatisticsCollector(boolean openStatisticsCollector) {
+        this.openStatisticsCollector = openStatisticsCollector;
+    }
+
+    public long getStatisticsCallbackIntervalSeconds() {
+        return statisticsCallbackIntervalSeconds;
+    }
+
+    public void setStatisticsCallbackIntervalSeconds(long statisticsCallbackIntervalSeconds) {
+        this.statisticsCallbackIntervalSeconds = statisticsCallbackIntervalSeconds;
+    }
+
+    public String getDbPath() {
+        return dbPath;
+    }
+
+    public void setDbPath(String dbPath) {
+        this.dbPath = dbPath;
+    }
+
+    @Override
+    public String toString() {
+        return "RocksDBOptions{" + "sync=" + sync + ", disableWAL=" + disableWAL + ", fastSnapshot=" + fastSnapshot
+               + ", asyncSnapshot=" + asyncSnapshot + ", openStatisticsCollector=" + openStatisticsCollector
+               + ", statisticsCallbackIntervalSeconds=" + statisticsCallbackIntervalSeconds + ", dbPath='" + dbPath
+               + '\'' + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RpcOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RpcOptions.java
new file mode 100644
index 0000000..2b39880
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/RpcOptions.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RpcOptions {
+
+    private int callbackExecutorCorePoolSize    = Utils.cpus() << 2;
+    private int callbackExecutorMaximumPoolSize = Utils.cpus() << 3;
+    private int callbackExecutorQueueCapacity   = 512;
+    private int rpcTimeoutMillis                = 5000;
+
+    public int getCallbackExecutorCorePoolSize() {
+        return callbackExecutorCorePoolSize;
+    }
+
+    public void setCallbackExecutorCorePoolSize(int callbackExecutorCorePoolSize) {
+        this.callbackExecutorCorePoolSize = callbackExecutorCorePoolSize;
+    }
+
+    public int getCallbackExecutorMaximumPoolSize() {
+        return callbackExecutorMaximumPoolSize;
+    }
+
+    public void setCallbackExecutorMaximumPoolSize(int callbackExecutorMaximumPoolSize) {
+        this.callbackExecutorMaximumPoolSize = callbackExecutorMaximumPoolSize;
+    }
+
+    public int getCallbackExecutorQueueCapacity() {
+        return callbackExecutorQueueCapacity;
+    }
+
+    public void setCallbackExecutorQueueCapacity(int callbackExecutorQueueCapacity) {
+        this.callbackExecutorQueueCapacity = callbackExecutorQueueCapacity;
+    }
+
+    public int getRpcTimeoutMillis() {
+        return rpcTimeoutMillis;
+    }
+
+    public void setRpcTimeoutMillis(int rpcTimeoutMillis) {
+        this.rpcTimeoutMillis = rpcTimeoutMillis;
+    }
+
+    @Override
+    public String toString() {
+        return "RpcOptions{" + "callbackExecutorCorePoolSize=" + callbackExecutorCorePoolSize
+               + ", callbackExecutorMaximumPoolSize=" + callbackExecutorMaximumPoolSize
+               + ", callbackExecutorQueueCapacity=" + callbackExecutorQueueCapacity + ", rpcTimeoutMillis="
+               + rpcTimeoutMillis + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/StoreEngineOptions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/StoreEngineOptions.java
new file mode 100644
index 0000000..b5190d5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/StoreEngineOptions.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StoreEngineOptions {
+
+    private StorageType               storageType                   = StorageType.RocksDB;
+    private RocksDBOptions            rocksDBOptions;
+    private MemoryDBOptions           memoryDBOptions;
+    private String                    raftDataPath;
+    private Endpoint                  serverAddress;
+    // Most configurations do not need to be configured separately for each raft-group,
+    // so a common configuration is provided, and each raft-group can copy from here.
+    private NodeOptions               commonNodeOptions             = new NodeOptions();
+    private List regionEngineOptionsList;
+    private String                    initialServerList;
+    private HeartbeatOptions          heartbeatOptions;
+    private boolean                   useSharedRpcExecutor;
+    // thread poll number of threads
+    private int                       readIndexCoreThreads          = Math.max(Utils.cpus() << 2, 16);
+    private int                       leaderStateTriggerCoreThreads = 4;
+    private int                       snapshotCoreThreads           = 1;
+    private int                       snapshotMaxThreads            = 32;
+    private int                       cliRpcCoreThreads             = Utils.cpus() << 2;
+    private int                       raftRpcCoreThreads            = Math.max(Utils.cpus() << 3, 32);
+    private int                       kvRpcCoreThreads              = Math.max(Utils.cpus() << 3, 32);
+    // metrics schedule option (seconds), won't start reporter id metricsReportPeriod <= 0
+    private long                      metricsReportPeriod           = TimeUnit.MINUTES.toSeconds(5);
+    // the minimum number of keys required to split, less than this value will refuse to split
+    private long                      leastKeysOnSplit              = 10000;
+
+    public StorageType getStorageType() {
+        return storageType;
+    }
+
+    public void setStorageType(StorageType storageType) {
+        this.storageType = storageType;
+    }
+
+    public RocksDBOptions getRocksDBOptions() {
+        return rocksDBOptions;
+    }
+
+    public void setRocksDBOptions(RocksDBOptions rocksDBOptions) {
+        this.rocksDBOptions = rocksDBOptions;
+    }
+
+    public MemoryDBOptions getMemoryDBOptions() {
+        return memoryDBOptions;
+    }
+
+    public void setMemoryDBOptions(MemoryDBOptions memoryDBOptions) {
+        this.memoryDBOptions = memoryDBOptions;
+    }
+
+    public String getRaftDataPath() {
+        return raftDataPath;
+    }
+
+    public void setRaftDataPath(String raftDataPath) {
+        this.raftDataPath = raftDataPath;
+    }
+
+    public Endpoint getServerAddress() {
+        return serverAddress;
+    }
+
+    public void setServerAddress(Endpoint serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    public NodeOptions getCommonNodeOptions() {
+        return commonNodeOptions;
+    }
+
+    public void setCommonNodeOptions(NodeOptions commonNodeOptions) {
+        this.commonNodeOptions = commonNodeOptions;
+    }
+
+    public List getRegionEngineOptionsList() {
+        return regionEngineOptionsList;
+    }
+
+    public void setRegionEngineOptionsList(List regionEngineOptionsList) {
+        this.regionEngineOptionsList = regionEngineOptionsList;
+    }
+
+    public String getInitialServerList() {
+        return initialServerList;
+    }
+
+    public void setInitialServerList(String initialServerList) {
+        this.initialServerList = initialServerList;
+    }
+
+    public HeartbeatOptions getHeartbeatOptions() {
+        return heartbeatOptions;
+    }
+
+    public void setHeartbeatOptions(HeartbeatOptions heartbeatOptions) {
+        this.heartbeatOptions = heartbeatOptions;
+    }
+
+    public boolean isUseSharedRpcExecutor() {
+        return useSharedRpcExecutor;
+    }
+
+    public void setUseSharedRpcExecutor(boolean useSharedRpcExecutor) {
+        this.useSharedRpcExecutor = useSharedRpcExecutor;
+    }
+
+    public int getReadIndexCoreThreads() {
+        return readIndexCoreThreads;
+    }
+
+    public void setReadIndexCoreThreads(int readIndexCoreThreads) {
+        this.readIndexCoreThreads = readIndexCoreThreads;
+    }
+
+    public int getLeaderStateTriggerCoreThreads() {
+        return leaderStateTriggerCoreThreads;
+    }
+
+    public void setLeaderStateTriggerCoreThreads(int leaderStateTriggerCoreThreads) {
+        this.leaderStateTriggerCoreThreads = leaderStateTriggerCoreThreads;
+    }
+
+    public int getSnapshotCoreThreads() {
+        return snapshotCoreThreads;
+    }
+
+    public void setSnapshotCoreThreads(int snapshotCoreThreads) {
+        this.snapshotCoreThreads = snapshotCoreThreads;
+    }
+
+    public int getSnapshotMaxThreads() {
+        return snapshotMaxThreads;
+    }
+
+    public void setSnapshotMaxThreads(int snapshotMaxThreads) {
+        this.snapshotMaxThreads = snapshotMaxThreads;
+    }
+
+    public int getCliRpcCoreThreads() {
+        return cliRpcCoreThreads;
+    }
+
+    public void setCliRpcCoreThreads(int cliRpcCoreThreads) {
+        this.cliRpcCoreThreads = cliRpcCoreThreads;
+    }
+
+    public int getRaftRpcCoreThreads() {
+        return raftRpcCoreThreads;
+    }
+
+    public void setRaftRpcCoreThreads(int raftRpcCoreThreads) {
+        this.raftRpcCoreThreads = raftRpcCoreThreads;
+    }
+
+    public int getKvRpcCoreThreads() {
+        return kvRpcCoreThreads;
+    }
+
+    public void setKvRpcCoreThreads(int kvRpcCoreThreads) {
+        this.kvRpcCoreThreads = kvRpcCoreThreads;
+    }
+
+    public long getMetricsReportPeriod() {
+        return metricsReportPeriod;
+    }
+
+    public void setMetricsReportPeriod(long metricsReportPeriod) {
+        this.metricsReportPeriod = metricsReportPeriod;
+    }
+
+    public long getLeastKeysOnSplit() {
+        return leastKeysOnSplit;
+    }
+
+    public void setLeastKeysOnSplit(long leastKeysOnSplit) {
+        this.leastKeysOnSplit = leastKeysOnSplit;
+    }
+
+    @Override
+    public String toString() {
+        return "StoreEngineOptions{" + "storageType=" + storageType + ", rocksDBOptions=" + rocksDBOptions
+               + ", memoryDBOptions=" + memoryDBOptions + ", raftDataPath='" + raftDataPath + '\'' + ", serverAddress="
+               + serverAddress + ", commonNodeOptions=" + commonNodeOptions + ", regionEngineOptionsList="
+               + regionEngineOptionsList + ", initialServerList='" + initialServerList + '\'' + ", heartbeatOptions="
+               + heartbeatOptions + ", useSharedRpcExecutor=" + useSharedRpcExecutor + ", readIndexCoreThreads="
+               + readIndexCoreThreads + ", leaderStateTriggerCoreThreads=" + leaderStateTriggerCoreThreads
+               + ", snapshotCoreThreads=" + snapshotCoreThreads + ", snapshotMaxThreads=" + snapshotMaxThreads
+               + ", cliRpcCoreThreads=" + cliRpcCoreThreads + ", raftRpcCoreThreads=" + raftRpcCoreThreads
+               + ", kvRpcCoreThreads=" + kvRpcCoreThreads + ", metricsReportPeriod=" + metricsReportPeriod
+               + ", leastKeysOnSplit=" + leastKeysOnSplit + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/BatchingOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/BatchingOptionsConfigured.java
new file mode 100644
index 0000000..1c5f30d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/BatchingOptionsConfigured.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.BatchingOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class BatchingOptionsConfigured implements Configured {
+
+    private final BatchingOptions opts;
+
+    public static BatchingOptionsConfigured newConfigured() {
+        return new BatchingOptionsConfigured(new BatchingOptions());
+    }
+
+    public static BatchingOptions newDefaultConfig() {
+        return new BatchingOptions();
+    }
+
+    public BatchingOptionsConfigured withAllowBatching(final boolean allowBatching) {
+        this.opts.setAllowBatching(allowBatching);
+        return this;
+    }
+
+    public BatchingOptionsConfigured withBatchSize(final int batchSize) {
+        this.opts.setBatchSize(batchSize);
+        return this;
+    }
+
+    public BatchingOptionsConfigured withBufSize(final int bufSize) {
+        this.opts.setBufSize(bufSize);
+        return this;
+    }
+
+    public BatchingOptionsConfigured withMaxWriteBytes(final int maxWriteBytes) {
+        this.opts.setMaxWriteBytes(maxWriteBytes);
+        return this;
+    }
+
+    public BatchingOptionsConfigured withMaxReadBytes(final int maxReadBytes) {
+        this.opts.setMaxReadBytes(maxReadBytes);
+        return this;
+    }
+
+    @Override
+    public BatchingOptions config() {
+        return this.opts;
+    }
+
+    private BatchingOptionsConfigured(BatchingOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/HeartbeatOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/HeartbeatOptionsConfigured.java
new file mode 100644
index 0000000..edec1aa
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/HeartbeatOptionsConfigured.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.HeartbeatOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class HeartbeatOptionsConfigured implements Configured {
+
+    private final HeartbeatOptions opts;
+
+    public static HeartbeatOptionsConfigured newConfigured() {
+        return new HeartbeatOptionsConfigured(new HeartbeatOptions());
+    }
+
+    public HeartbeatOptionsConfigured withStoreHeartbeatIntervalSeconds(final long storeHeartbeatIntervalSeconds) {
+        this.opts.setStoreHeartbeatIntervalSeconds(storeHeartbeatIntervalSeconds);
+        return this;
+    }
+
+    public HeartbeatOptionsConfigured withRegionHeartbeatIntervalSeconds(final long regionHeartbeatIntervalSeconds) {
+        this.opts.setRegionHeartbeatIntervalSeconds(regionHeartbeatIntervalSeconds);
+        return this;
+    }
+
+    public HeartbeatOptionsConfigured withHeartbeatRpcTimeoutMillis(final int heartbeatRpcTimeoutMillis) {
+        this.opts.setHeartbeatRpcTimeoutMillis(heartbeatRpcTimeoutMillis);
+        return this;
+    }
+
+    @Override
+    public HeartbeatOptions config() {
+        return this.opts;
+    }
+
+    private HeartbeatOptionsConfigured(HeartbeatOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MemoryDBOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MemoryDBOptionsConfigured.java
new file mode 100644
index 0000000..d4a2fef
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MemoryDBOptionsConfigured.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class MemoryDBOptionsConfigured implements Configured {
+
+    private final MemoryDBOptions opts;
+
+    public static MemoryDBOptionsConfigured newConfigured() {
+        return new MemoryDBOptionsConfigured(new MemoryDBOptions());
+    }
+
+    public MemoryDBOptionsConfigured withKeysPerSegment(final int keysPerSegment) {
+        this.opts.setKeysPerSegment(keysPerSegment);
+        return this;
+    }
+
+    @Override
+    public MemoryDBOptions config() {
+        return this.opts;
+    }
+
+    private MemoryDBOptionsConfigured(MemoryDBOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionEngineOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionEngineOptionsConfigured.java
new file mode 100644
index 0000000..ad3cace
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionEngineOptionsConfigured.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class MultiRegionEngineOptionsConfigured implements Configured> {
+
+    private final Map optsTable;
+
+    public static MultiRegionEngineOptionsConfigured newConfigured() {
+        return new MultiRegionEngineOptionsConfigured(Maps.newHashMap());
+    }
+
+    public MultiRegionEngineOptionsConfigured withStartKey(final Long regionId, final String startKey) {
+        getOrCreateOptsById(regionId).setStartKey(startKey);
+        return this;
+    }
+
+    public MultiRegionEngineOptionsConfigured withStartKeyBytes(final Long regionId, final byte[] startKeyBytes) {
+        getOrCreateOptsById(regionId).setStartKeyBytes(startKeyBytes);
+        return this;
+    }
+
+    public MultiRegionEngineOptionsConfigured withEndKey(final Long regionId, final String endKey) {
+        getOrCreateOptsById(regionId).setEndKey(endKey);
+        return this;
+    }
+
+    public MultiRegionEngineOptionsConfigured withEndKeyBytes(final Long regionId, final byte[] endKeyBytes) {
+        getOrCreateOptsById(regionId).setEndKeyBytes(endKeyBytes);
+        return this;
+    }
+
+    public MultiRegionEngineOptionsConfigured withNodeOptions(final Long regionId, final NodeOptions nodeOptions) {
+        getOrCreateOptsById(regionId).setNodeOptions(nodeOptions);
+        return this;
+    }
+
+    public MultiRegionEngineOptionsConfigured withMetricsReportPeriod(final Long regionId,
+                                                                      final long metricsReportPeriod) {
+        getOrCreateOptsById(regionId).setMetricsReportPeriod(metricsReportPeriod);
+        return this;
+    }
+
+    @Override
+    public List config() {
+        return Lists.newArrayList(this.optsTable.values());
+    }
+
+    private RegionEngineOptions getOrCreateOptsById(final Long regionId) {
+        Requires.requireNonNull(regionId, "regionId");
+        RegionEngineOptions opts = this.optsTable.get(regionId);
+        if (opts != null) {
+            return opts;
+        }
+        opts = new RegionEngineOptions();
+        opts.setRegionId(regionId);
+        this.optsTable.put(regionId, opts);
+        return opts;
+    }
+
+    private MultiRegionEngineOptionsConfigured(Map optsTable) {
+        this.optsTable = optsTable;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionRouteTableOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionRouteTableOptionsConfigured.java
new file mode 100644
index 0000000..6110de1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/MultiRegionRouteTableOptionsConfigured.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.rhea.options.RegionRouteTableOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class MultiRegionRouteTableOptionsConfigured implements Configured> {
+
+    private final Map optsTable;
+
+    public static MultiRegionRouteTableOptionsConfigured newConfigured() {
+        return new MultiRegionRouteTableOptionsConfigured(Maps.newHashMap());
+    }
+
+    public MultiRegionRouteTableOptionsConfigured withStartKey(final Long regionId, final String startKey) {
+        getOrCreateOptsById(regionId).setStartKey(startKey);
+        return this;
+    }
+
+    public MultiRegionRouteTableOptionsConfigured withStartKeyBytes(final Long regionId, final byte[] startKeyBytes) {
+        getOrCreateOptsById(regionId).setStartKeyBytes(startKeyBytes);
+        return this;
+    }
+
+    public MultiRegionRouteTableOptionsConfigured withEndKey(final Long regionId, final String endKey) {
+        getOrCreateOptsById(regionId).setEndKey(endKey);
+        return this;
+    }
+
+    public MultiRegionRouteTableOptionsConfigured withEndKeyBytes(final Long regionId, final byte[] endKeyBytes) {
+        getOrCreateOptsById(regionId).setEndKeyBytes(endKeyBytes);
+        return this;
+    }
+
+    public MultiRegionRouteTableOptionsConfigured withInitialServerList(final Long regionId,
+                                                                        final String initialServerList) {
+        getOrCreateOptsById(regionId).setInitialServerList(initialServerList);
+        return this;
+    }
+
+    @Override
+    public List config() {
+        return Lists.newArrayList(this.optsTable.values());
+    }
+
+    private RegionRouteTableOptions getOrCreateOptsById(final Long regionId) {
+        Requires.requireNonNull(regionId, "regionId");
+        RegionRouteTableOptions opts = this.optsTable.get(regionId);
+        if (opts != null) {
+            return opts;
+        }
+        opts = new RegionRouteTableOptions();
+        opts.setRegionId(regionId);
+        this.optsTable.put(regionId, opts);
+        return opts;
+    }
+
+    public MultiRegionRouteTableOptionsConfigured(Map optsTable) {
+        this.optsTable = optsTable;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverOptionsConfigured.java
new file mode 100644
index 0000000..6cdc1d0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverOptionsConfigured.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionRouteTableOptions;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class PlacementDriverOptionsConfigured implements Configured {
+
+    private final PlacementDriverOptions opts;
+
+    public static PlacementDriverOptionsConfigured newConfigured() {
+        return new PlacementDriverOptionsConfigured(new PlacementDriverOptions());
+    }
+
+    public PlacementDriverOptionsConfigured withFake(final boolean fake) {
+        this.opts.setFake(fake);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withCliOptions(final CliOptions cliOptions) {
+        this.opts.setCliOptions(cliOptions);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withPdRpcOptions(final RpcOptions pdRpcOptions) {
+        this.opts.setPdRpcOptions(pdRpcOptions);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withPdGroupId(final String pdGroupId) {
+        this.opts.setPdGroupId(pdGroupId);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withRegionRouteTableOptionsList(final List regionRouteTableOptionsList) {
+        this.opts.setRegionRouteTableOptionsList(regionRouteTableOptionsList);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withInitialServerList(final String initialServerList) {
+        this.opts.setInitialServerList(initialServerList);
+        return this;
+    }
+
+    public PlacementDriverOptionsConfigured withInitialPdServerList(final String initialPdServerList) {
+        this.opts.setInitialPdServerList(initialPdServerList);
+        return this;
+    }
+
+    @Override
+    public PlacementDriverOptions config() {
+        return this.opts;
+    }
+
+    private PlacementDriverOptionsConfigured(PlacementDriverOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RheaKVStoreOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RheaKVStoreOptionsConfigured.java
new file mode 100644
index 0000000..6e8be7e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RheaKVStoreOptionsConfigured.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.BatchingOptions;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class RheaKVStoreOptionsConfigured implements Configured {
+
+    private final RheaKVStoreOptions opts;
+
+    public static RheaKVStoreOptionsConfigured newConfigured() {
+        return new RheaKVStoreOptionsConfigured(new RheaKVStoreOptions());
+    }
+
+    public RheaKVStoreOptionsConfigured withClusterId(final long clusterId) {
+        this.opts.setClusterId(clusterId);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withClusterName(final String clusterName) {
+        this.opts.setClusterName(clusterName);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withPlacementDriverOptions(final PlacementDriverOptions placementDriverOptions) {
+        this.opts.setPlacementDriverOptions(placementDriverOptions);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withStoreEngineOptions(final StoreEngineOptions storeEngineOptions) {
+        this.opts.setStoreEngineOptions(storeEngineOptions);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withInitialServerList(final String initialServerList) {
+        this.opts.setInitialServerList(initialServerList);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withOnlyLeaderRead(final boolean onlyLeaderRead) {
+        this.opts.setOnlyLeaderRead(onlyLeaderRead);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withRpcOptions(final RpcOptions rpcOptions) {
+        this.opts.setRpcOptions(rpcOptions);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withFailoverRetries(final int failoverRetries) {
+        this.opts.setFailoverRetries(failoverRetries);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withFutureTimeoutMillis(final long futureTimeoutMillis) {
+        this.opts.setFutureTimeoutMillis(futureTimeoutMillis);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withUseParallelKVExecutor(final boolean useParallelKVExecutor) {
+        this.opts.setUseParallelKVExecutor(useParallelKVExecutor);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withBatchingOptions(final BatchingOptions batchingOptions) {
+        this.opts.setBatchingOptions(batchingOptions);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withUseParallelCompress(final boolean useParallelCompress) {
+        this.opts.setUseParallelCompress(useParallelCompress);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withCompressThreads(final int compressThreads) {
+        this.opts.setCompressThreads(compressThreads);
+        return this;
+    }
+
+    public RheaKVStoreOptionsConfigured withDeCompressThreads(final int deCompressThreads) {
+        this.opts.setDeCompressThreads(deCompressThreads);
+        return this;
+    }
+
+    @Override
+    public RheaKVStoreOptions config() {
+        return this.opts;
+    }
+
+    private RheaKVStoreOptionsConfigured(RheaKVStoreOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RocksDBOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RocksDBOptionsConfigured.java
new file mode 100644
index 0000000..57c3b43
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RocksDBOptionsConfigured.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class RocksDBOptionsConfigured implements Configured {
+
+    private final RocksDBOptions opts;
+
+    public static RocksDBOptionsConfigured newConfigured() {
+        return new RocksDBOptionsConfigured(new RocksDBOptions());
+    }
+
+    public RocksDBOptionsConfigured withSync(final boolean sync) {
+        this.opts.setSync(sync);
+        return this;
+    }
+
+    public RocksDBOptionsConfigured withFastSnapshot(final boolean fastSnapshot) {
+        this.opts.setFastSnapshot(fastSnapshot);
+        return this;
+    }
+
+    public RocksDBOptionsConfigured withOpenStatisticsCollector(final boolean openStatisticsCollector) {
+        this.opts.setOpenStatisticsCollector(openStatisticsCollector);
+        return this;
+    }
+
+    public RocksDBOptionsConfigured withStatisticsCallbackIntervalSeconds(final long statisticsCallbackIntervalSeconds) {
+        this.opts.setStatisticsCallbackIntervalSeconds(statisticsCallbackIntervalSeconds);
+        return this;
+    }
+
+    public RocksDBOptionsConfigured withDbPath(final String dbPath) {
+        this.opts.setDbPath(dbPath);
+        return this;
+    }
+
+    @Override
+    public RocksDBOptions config() {
+        return this.opts;
+    }
+
+    private RocksDBOptionsConfigured(RocksDBOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RpcOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RpcOptionsConfigured.java
new file mode 100644
index 0000000..785fbdb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/RpcOptionsConfigured.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.RpcOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class RpcOptionsConfigured implements Configured {
+
+    private final RpcOptions opts;
+
+    public static RpcOptionsConfigured newConfigured() {
+        return new RpcOptionsConfigured(new RpcOptions());
+    }
+
+    public static RpcOptions newDefaultConfig() {
+        return new RpcOptions();
+    }
+
+    @Override
+    public RpcOptions config() {
+        return this.opts;
+    }
+
+    private RpcOptionsConfigured(RpcOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/StoreEngineOptionsConfigured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/StoreEngineOptionsConfigured.java
new file mode 100644
index 0000000..f297b28
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/StoreEngineOptionsConfigured.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.options.HeartbeatOptions;
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class StoreEngineOptionsConfigured implements Configured {
+
+    private final StoreEngineOptions opts;
+
+    public static StoreEngineOptionsConfigured newConfigured() {
+        return new StoreEngineOptionsConfigured(new StoreEngineOptions());
+    }
+
+    public StoreEngineOptionsConfigured withStorageType(final StorageType storageType) {
+        this.opts.setStorageType(storageType);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withRocksDBOptions(final RocksDBOptions rocksDBOptions) {
+        this.opts.setRocksDBOptions(rocksDBOptions);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withMemoryDBOptions(final MemoryDBOptions memoryDBOptions) {
+        this.opts.setMemoryDBOptions(memoryDBOptions);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withRaftDataPath(final String raftDataPath) {
+        this.opts.setRaftDataPath(raftDataPath);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withServerAddress(final Endpoint serverAddress) {
+        this.opts.setServerAddress(serverAddress);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withCommonNodeOptions(final NodeOptions nodeOptions) {
+        this.opts.setCommonNodeOptions(nodeOptions);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withRegionEngineOptionsList(final List regionEngineOptionsList) {
+        this.opts.setRegionEngineOptionsList(regionEngineOptionsList);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withHeartbeatOptions(final HeartbeatOptions heartbeatOptions) {
+        this.opts.setHeartbeatOptions(heartbeatOptions);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withUseSharedRpcExecutor(final boolean useSharedRpcExecutor) {
+        this.opts.setUseSharedRpcExecutor(useSharedRpcExecutor);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withReadIndexCoreThreads(final int readIndexCoreThreads) {
+        this.opts.setReadIndexCoreThreads(readIndexCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withLeaderStateTriggerCoreThreads(final int leaderStateTriggerCoreThreads) {
+        this.opts.setLeaderStateTriggerCoreThreads(leaderStateTriggerCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withSnapshotCoreThreads(final int snapshotCoreThreads) {
+        this.opts.setSnapshotCoreThreads(snapshotCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withCliRpcCoreThreads(final int cliRpcCoreThreads) {
+        this.opts.setCliRpcCoreThreads(cliRpcCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withRaftRpcCoreThreads(final int raftRpcCoreThreads) {
+        this.opts.setRaftRpcCoreThreads(raftRpcCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withKvRpcCoreThreads(final int kvRpcCoreThreads) {
+        this.opts.setKvRpcCoreThreads(kvRpcCoreThreads);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withMetricsReportPeriod(final long metricsReportPeriod) {
+        this.opts.setMetricsReportPeriod(metricsReportPeriod);
+        return this;
+    }
+
+    public StoreEngineOptionsConfigured withLeastKeysOnSplit(final long leastKeysOnSplit) {
+        this.opts.setLeastKeysOnSplit(leastKeysOnSplit);
+        return this;
+    }
+
+    @Override
+    public StoreEngineOptions config() {
+        return this.opts;
+    }
+
+    private StoreEngineOptionsConfigured(StoreEngineOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatistics.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatistics.java
new file mode 100644
index 0000000..28cf3e9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatistics.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.rocks.support;
+
+import org.rocksdb.HistogramData;
+import org.rocksdb.HistogramType;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.Statistics;
+import org.rocksdb.TickerType;
+
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.util.DebugStatistics;
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class RocksStatistics {
+
+    private static final ReferenceFieldUpdater statisticsGetter = Updaters
+                                                                                                      .newReferenceFieldUpdater(
+                                                                                                          RocksRawKVStore.class,
+                                                                                                          "statistics");
+    private static final ReferenceFieldUpdater         dbGetter         = Updaters
+                                                                                                      .newReferenceFieldUpdater(
+                                                                                                          RocksRawKVStore.class,
+                                                                                                          "db");
+
+    /**
+     * Get the count for a ticker.
+     */
+    public static long getTickerCount(final RocksRawKVStore rocksRawKVStore, final TickerType tickerType) {
+        final Statistics statistics = statistics(rocksRawKVStore);
+        if (statistics == null) {
+            return -1L;
+        }
+        return statistics.getTickerCount(tickerType);
+    }
+
+    /**
+     * Get the count for a ticker and reset the tickers count.
+     */
+    public static long getAndResetTickerCount(final RocksRawKVStore rocksRawKVStore, final TickerType tickerType) {
+        final Statistics statistics = statistics(rocksRawKVStore);
+        if (statistics == null) {
+            return -1L;
+        }
+        return statistics.getAndResetTickerCount(tickerType);
+    }
+
+    /**
+     * Gets the histogram data for a particular histogram.
+     */
+    public static HistogramData getHistogramData(final RocksRawKVStore rocksRawKVStore,
+                                                 final HistogramType histogramType) {
+        final Statistics statistics = statistics(rocksRawKVStore);
+        if (statistics == null) {
+            return null;
+        }
+        return statistics.getHistogramData(histogramType);
+    }
+
+    /**
+     * Gets a string representation of a particular histogram.
+     */
+    public String getHistogramString(final RocksRawKVStore rocksRawKVStore, final HistogramType histogramType) {
+        final Statistics statistics = statistics(rocksRawKVStore);
+        if (statistics == null) {
+            return "";
+        }
+        return statistics.getHistogramString(histogramType);
+    }
+
+    /**
+     * String representation of the statistic.
+     */
+    public static String getStatisticsString(final RocksRawKVStore rocksRawKVStore) {
+        final Statistics statistics = statistics(rocksRawKVStore);
+        if (statistics == null) {
+            return "";
+        }
+        return statistics.toString();
+    }
+
+    /**
+     * DB implementations can export properties about their state
+     * via this method.
+     */
+    public static String getProperty(final RocksRawKVStore rocksRawKVStore, final String name) {
+        final RocksDB db = db(rocksRawKVStore);
+        if (db != null) {
+            try {
+                return db.getProperty(name);
+            } catch (final RocksDBException e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        throw new NullPointerException("db");
+    }
+
+    private static Statistics statistics(final RocksRawKVStore rocksRawKVStore) {
+        return statisticsGetter.get(rocksRawKVStore);
+    }
+
+    private static RocksDB db(final RocksRawKVStore rocksRawKVStore) {
+        return dbGetter.get(rocksRawKVStore);
+    }
+
+    private RocksStatistics() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatisticsCollector.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatisticsCollector.java
new file mode 100644
index 0000000..aecb18f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rocks/support/RocksStatisticsCollector.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.rocks.support;
+
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.rocksdb.HistogramData;
+import org.rocksdb.HistogramType;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatisticsCollectorCallback;
+import org.rocksdb.StatsCollectorInput;
+import org.rocksdb.TickerType;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+
+/**
+ * Helper class to collect rocksDB statistics periodically at a period specified
+ * in constructor. Callback function (provided in constructor) is called with
+ * every statistics collection.
+ * 

+ * Caller should call start() to start statistics collection. Shutdown() should
+ * be called to stop stats collection and should be called before statistics (
+ * provided in constructor) reference has been disposed.
+ *
+ * @author jiachun.fjc
+ */
+public class RocksStatisticsCollector {
+
+    private final CopyOnWriteArrayList statsCollectorInputList = new CopyOnWriteArrayList<>();
+    private final long                                      statsCollectionIntervalInMillis;
+    private final ExecutorService                           executorService;
+    private volatile boolean                                isRunning               = true;
+
+    public RocksStatisticsCollector(final long statsCollectionIntervalInMillis) {
+        this.statsCollectionIntervalInMillis = statsCollectionIntervalInMillis;
+        this.executorService = Executors.newSingleThreadExecutor(new NamedThreadFactory("rocks-statistics-collector",
+            true));
+    }
+
+    public void start() {
+        this.executorService.submit(collectStatistics());
+    }
+
+    public void addStatsCollectorInput(final StatsCollectorInput input) {
+        statsCollectorInputList.add(input);
+    }
+
+    /**
+     * Shuts down statistics collector.
+     *
+     * @param shutdownTimeout Time in milli-seconds to wait for shutdown before
+     *                        killing the collection process.
+     * @throws InterruptedException thrown if Threads are interrupted.
+     */
+    public void shutdown(final int shutdownTimeout) throws InterruptedException {
+        this.isRunning = false;
+
+        this.executorService.shutdownNow();
+        // Wait for collectStatistics runnable to finish so that disposal of
+        // statistics does not cause any exceptions to be thrown.
+        this.executorService.awaitTermination(shutdownTimeout, TimeUnit.MILLISECONDS);
+    }
+
+    private Runnable collectStatistics() {
+        return () -> {
+            while (this.isRunning) {
+                try {
+                    if (Thread.currentThread().isInterrupted()) {
+                        break;
+                    }
+                    for (final StatsCollectorInput statsCollectorInput : this.statsCollectorInputList) {
+                        final Statistics statistics = statsCollectorInput.getStatistics();
+                        final StatisticsCollectorCallback statsCallback = statsCollectorInput.getCallback();
+                        // Collect ticker data
+                        for (final TickerType ticker : TickerType.values()) {
+                            if (ticker != TickerType.TICKER_ENUM_MAX) {
+                                long tickerValue = statistics.getTickerCount(ticker);
+                                statsCallback.tickerCallback(ticker, tickerValue);
+                            }
+                        }
+                        // Collect histogram data
+                        for (final HistogramType histogramType : HistogramType.values()) {
+                            if (histogramType != HistogramType.HISTOGRAM_ENUM_MAX) {
+                                HistogramData histogramData = statistics.getHistogramData(histogramType);
+                                statsCallback.histogramCallback(histogramType, histogramData);
+                            }
+                        }
+                    }
+                    Thread.sleep(this.statsCollectionIntervalInMillis);
+                } catch (final InterruptedException e) {
+                    Thread.currentThread().interrupt();
+                    break;
+                } catch (final Exception e) {
+                    throw new RuntimeException("Error while calculating statistics", e);
+                }
+            }
+        };
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ExtSerializerSupports.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ExtSerializerSupports.java
new file mode 100644
index 0000000..08999b9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ExtSerializerSupports.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.rpc;
+
+import com.alipay.remoting.InvokeContext;
+import com.alipay.remoting.serialization.SerializerManager;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class ExtSerializerSupports {
+
+    private static final InvokeContext INVOKE_CONTEXT = new InvokeContext();
+
+    public static byte                 PROTO_STUFF    = 2;
+
+    static {
+        SerializerManager.addSerializer(PROTO_STUFF, ProtostuffSerializer.INSTANCE);
+        INVOKE_CONTEXT.put(InvokeContext.BOLT_CUSTOM_SERIALIZER, PROTO_STUFF);
+        INVOKE_CONTEXT.put(InvokeContext.BOLT_CRC_SWITCH, false);
+    }
+
+    public static void init() {
+        // Will execute the code first of the static block
+    }
+
+    public static InvokeContext getInvokeContext() {
+        return INVOKE_CONTEXT;
+    }
+
+    private ExtSerializerSupports() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ProtostuffSerializer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ProtostuffSerializer.java
new file mode 100644
index 0000000..97b5fe1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/rpc/ProtostuffSerializer.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.rpc;
+
+import java.util.concurrent.ConcurrentMap;
+
+import com.alipay.remoting.exception.CodecException;
+import com.alipay.remoting.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ProtostuffSerializer implements Serializer {
+
+    public static final ProtostuffSerializer                          INSTANCE   = new ProtostuffSerializer();
+
+    private static final ConcurrentMap>              classCache = Maps.newConcurrentMap();
+
+    private final com.alipay.sofa.jraft.rhea.serialization.Serializer delegate   = Serializers
+                                                                                     .getSerializer(Serializers.PROTO_STUFF);
+
+    @Override
+    public byte[] serialize(final Object obj) throws CodecException {
+        return this.delegate.writeObject(obj);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  T deserialize(final byte[] data, final String classOfT) throws CodecException {
+        Class clazz = classCache.get(classOfT);
+        if (clazz == null) {
+            try {
+                final Class newClazz = Class.forName(classOfT);
+                clazz = classCache.putIfAbsent(classOfT, newClazz);
+                if (clazz == null) {
+                    clazz = newClazz;
+                }
+            } catch (final Exception e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        return (T) this.delegate.readObject(data, clazz);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializer.java
new file mode 100644
index 0000000..12fd5f4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializer.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization;
+
+import java.nio.ByteBuffer;
+
+import com.alipay.sofa.jraft.rhea.serialization.io.InputBuf;
+import com.alipay.sofa.jraft.rhea.serialization.io.OutputBuf;
+
+/**
+ * This interface provides an abstract view for one or more serializer impl.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class Serializer {
+
+    /**
+     * The max buffer size for a {@link Serializer} to cached.
+     */
+    public static final int MAX_CACHED_BUF_SIZE = 256 * 1024;
+
+    /**
+     * The default buffer size for a {@link Serializer}.
+     */
+    public static final int DEFAULT_BUF_SIZE    = 512;
+
+    public abstract  OutputBuf writeObject(final OutputBuf outputBuf, final T obj);
+
+    public abstract  byte[] writeObject(final T obj);
+
+    public abstract  T readObject(final InputBuf inputBuf, final Class clazz);
+
+    public abstract  T readObject(final ByteBuffer buf, final Class clazz);
+
+    public abstract  T readObject(final byte[] bytes, final int offset, final int length, final Class clazz);
+
+    public  T readObject(final byte[] bytes, final Class clazz) {
+        return readObject(bytes, 0, bytes.length, clazz);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializers.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializers.java
new file mode 100644
index 0000000..9ece5b3
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/Serializers.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization;
+
+import com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.ProtoStuffSerializer;
+
+/**
+ * Holds all serializers.
+ *
+ * @author jiachun.fjc
+ */
+public final class Serializers {
+
+    public static final byte    PROTO_STUFF = 0x01;
+
+    private static Serializer[] serializers = new Serializer[5];
+
+    static {
+        addSerializer(PROTO_STUFF, new ProtoStuffSerializer());
+    }
+
+    public static Serializer getSerializer(final int idx) {
+        return serializers[idx];
+    }
+
+    public static Serializer getDefault() {
+        return serializers[PROTO_STUFF];
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private static void addSerializer(final int idx, final Serializer serializer) {
+        if (serializers.length <= idx) {
+            final Serializer[] newSerializers = new Serializer[idx + 5];
+            System.arraycopy(serializers, 0, newSerializers, 0, serializers.length);
+            serializers = newSerializers;
+        }
+        serializers[idx] = serializer;
+    }
+
+    private Serializers() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/ProtoStuffSerializer.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/ProtoStuffSerializer.java
new file mode 100644
index 0000000..ee2ca27
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/ProtoStuffSerializer.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import io.protostuff.Input;
+import io.protostuff.LinkedBuffer;
+import io.protostuff.Output;
+import io.protostuff.Schema;
+import io.protostuff.runtime.RuntimeSchema;
+
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io.Inputs;
+import com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io.LinkedBuffers;
+import com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io.Outputs;
+import com.alipay.sofa.jraft.rhea.serialization.io.InputBuf;
+import com.alipay.sofa.jraft.rhea.serialization.io.OutputBuf;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ProtoStuffSerializer extends Serializer {
+
+    static {
+        // see io.protostuff.runtime.RuntimeEnv
+
+        // If true, the constructor will always be obtained from {@code ReflectionFactory.newConstructorFromSerialization}.
+        //
+        // Enable this if you intend to avoid deserialize objects whose no-args constructor initializes (unwanted)
+        // internal state. This applies to complex/framework objects.
+        //
+        // If you intend to fill default field values using your default constructor, leave this disabled. This normally
+        // applies to java beans/data objects.
+        //
+        final String always_use_sun_reflection_factory = SystemPropertyUtil.get(
+            "rhea.serializer.protostuff.always_use_sun_reflection_factory", "false");
+        SystemPropertyUtil.setProperty("protostuff.runtime.always_use_sun_reflection_factory",
+            always_use_sun_reflection_factory);
+
+        // Disabled by default.  Writes a sentinel value (uint32) in place of null values.
+        //
+        // default is false
+        final String allow_null_array_element = SystemPropertyUtil.get(
+            "rhea.serializer.protostuff.allow_null_array_element", "false");
+        SystemPropertyUtil.setProperty("protostuff.runtime.allow_null_array_element", allow_null_array_element);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  OutputBuf writeObject(final OutputBuf outputBuf, final T obj) {
+        final Schema schema = RuntimeSchema.getSchema((Class) obj.getClass());
+
+        final Output output = Outputs.getOutput(outputBuf);
+        try {
+            schema.writeTo(output, obj);
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        }
+
+        return outputBuf;
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  byte[] writeObject(final T obj) {
+        final Schema schema = RuntimeSchema.getSchema((Class) obj.getClass());
+
+        final LinkedBuffer buf = LinkedBuffers.getLinkedBuffer();
+        final Output output = Outputs.getOutput(buf);
+        try {
+            schema.writeTo(output, obj);
+            return Outputs.toByteArray(output);
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        } finally {
+            LinkedBuffers.resetBuf(buf); // for reuse
+        }
+
+        return null; // never get here
+    }
+
+    @Override
+    public  T readObject(final InputBuf inputBuf, final Class clazz) {
+        final Schema schema = RuntimeSchema.getSchema(clazz);
+        final T msg = schema.newMessage();
+
+        final Input input = Inputs.getInput(inputBuf);
+        try {
+            schema.mergeFrom(input, msg);
+            Inputs.checkLastTagWas(input, 0);
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        } finally {
+            inputBuf.release();
+        }
+
+        return msg;
+    }
+
+    @Override
+    public  T readObject(final ByteBuffer buf, final Class clazz) {
+        final Schema schema = RuntimeSchema.getSchema(clazz);
+        final T msg = schema.newMessage();
+
+        final Input input = Inputs.getInput(buf);
+        try {
+            schema.mergeFrom(input, msg);
+            Inputs.checkLastTagWas(input, 0);
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        }
+
+        return msg;
+    }
+
+    @Override
+    public  T readObject(final byte[] bytes, final int offset, final int length, final Class clazz) {
+        final Schema schema = RuntimeSchema.getSchema(clazz);
+        final T msg = schema.newMessage();
+
+        final Input input = Inputs.getInput(bytes, offset, length);
+        try {
+            schema.mergeFrom(input, msg);
+            Inputs.checkLastTagWas(input, 0);
+        } catch (final IOException e) {
+            ThrowUtil.throwException(e);
+        }
+
+        return msg;
+    }
+
+    @Override
+    public String toString() {
+        return "proto_stuff";
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Inputs.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Inputs.java
new file mode 100644
index 0000000..134c87e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Inputs.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import java.nio.ByteBuffer;
+
+import io.protostuff.ByteArrayInput;
+import io.protostuff.Input;
+import io.protostuff.ProtobufException;
+
+import com.alipay.sofa.jraft.rhea.serialization.io.InputBuf;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class Inputs {
+
+    public static Input getInput(final InputBuf inputBuf) {
+        if (UnsafeUtil.hasUnsafe() && inputBuf.hasMemoryAddress()) {
+            return new UnsafeNioBufInput(inputBuf.nioByteBuffer(), true);
+        }
+        return new NioBufInput(inputBuf.nioByteBuffer(), true);
+    }
+
+    public static Input getInput(final ByteBuffer buf) {
+        if (UnsafeUtil.hasUnsafe() && buf.isDirect()) {
+            return new UnsafeNioBufInput(buf, true);
+        }
+        return new NioBufInput(buf, true);
+    }
+
+    public static Input getInput(final byte[] bytes, final int offset, final int length) {
+        return new ByteArrayInput(bytes, offset, length, true);
+    }
+
+    public static void checkLastTagWas(final Input input, final int value) throws ProtobufException {
+        if (input instanceof UnsafeNioBufInput) {
+            ((UnsafeNioBufInput) input).checkLastTagWas(value);
+        } else if (input instanceof NioBufInput) {
+            ((NioBufInput) input).checkLastTagWas(value);
+        } else if (input instanceof ByteArrayInput) {
+            ((ByteArrayInput) input).checkLastTagWas(value);
+        }
+    }
+
+    private Inputs() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/LinkedBuffers.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/LinkedBuffers.java
new file mode 100644
index 0000000..098e8a0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/LinkedBuffers.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import io.protostuff.LinkedBuffer;
+
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class LinkedBuffers {
+
+    // reuse the 'byte[]' of LinkedBuffer's head node
+    private static final ThreadLocal bufThreadLocal = ThreadLocal.withInitial(
+                                                                      () -> LinkedBuffer.allocate(Serializer.DEFAULT_BUF_SIZE));
+
+    public static LinkedBuffer getLinkedBuffer() {
+        return bufThreadLocal.get();
+    }
+
+    public static void resetBuf(final LinkedBuffer buf) {
+        buf.clear();
+    }
+
+    private LinkedBuffers() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufInput.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufInput.java
new file mode 100644
index 0000000..b519bd3
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufInput.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+
+import io.protostuff.ByteBufferInput;
+import io.protostuff.ByteString;
+import io.protostuff.Input;
+import io.protostuff.Output;
+import io.protostuff.ProtobufException;
+import io.protostuff.Schema;
+import io.protostuff.StringSerializer;
+import io.protostuff.UninitializedMessageException;
+
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtf8Util;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+import static io.protostuff.WireFormat.WIRETYPE_END_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED32;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED64;
+import static io.protostuff.WireFormat.WIRETYPE_LENGTH_DELIMITED;
+import static io.protostuff.WireFormat.WIRETYPE_START_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_TAIL_DELIMITER;
+import static io.protostuff.WireFormat.WIRETYPE_VARINT;
+import static io.protostuff.WireFormat.getTagFieldNumber;
+import static io.protostuff.WireFormat.getTagWireType;
+import static io.protostuff.WireFormat.makeTag;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+class NioBufInput implements Input {
+
+    static final int         TAG_TYPE_BITS = 3;
+    static final int         TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1;
+
+    static final Method      byteStringWrapMethod;
+
+    private final ByteBuffer nioBuffer;
+    private int              lastTag       = 0;
+    private int              packedLimit   = 0;
+
+    /**
+     * If true, the nested messages are group-encoded
+     */
+    public final boolean     decodeNestedMessageAsGroup;
+
+    /**
+     * An input for a ByteBuffer
+     *
+     * @param nioBuffer         the buffer to read from, it will not be sliced
+     * @param protostuffMessage if we are parsing a protostuff (true) or protobuf (false) message
+     */
+    public NioBufInput(ByteBuffer nioBuffer, boolean protostuffMessage) {
+        this.nioBuffer = nioBuffer;
+        this.decodeNestedMessageAsGroup = protostuffMessage;
+    }
+
+    /**
+     * Returns the current offset (the position).
+     */
+    public int currentOffset() {
+        return nioBuffer.position();
+    }
+
+    /**
+     * Returns the current limit (the end index).
+     */
+    public int currentLimit() {
+        return nioBuffer.limit();
+    }
+
+    /**
+     * Return true if currently reading packed field
+     */
+    public boolean isCurrentFieldPacked() {
+        return packedLimit != 0 && packedLimit != nioBuffer.position();
+    }
+
+    /**
+     * Returns the last tag.
+     */
+    public int getLastTag() {
+        return lastTag;
+    }
+
+    /**
+     * Attempt to read a field tag, returning zero if we have reached EOF. Protocol message parsers use this to read
+     * tags, since a protocol message may legally end wherever a tag occurs, and zero is not a valid tag number.
+     */
+    public int readTag() throws IOException {
+        if (!nioBuffer.hasRemaining()) {
+            lastTag = 0;
+            return 0;
+        }
+
+        final int tag = readRawVarInt32();
+        if (tag >>> TAG_TYPE_BITS == 0) {
+            // If we actually read zero, that's not a valid tag.
+            throw ProtocolException.invalidTag();
+        }
+        lastTag = tag;
+        return tag;
+    }
+
+    /**
+     * Verifies that the last call to readTag() returned the given tag value. This is used to verify that a nested group
+     * ended with the correct end tag.
+     *
+     * @throws ProtobufException {@code value} does not match the last tag.
+     */
+    public void checkLastTagWas(final int value) throws ProtobufException {
+        if (lastTag != value) {
+            throw ProtocolException.invalidEndTag();
+        }
+    }
+
+    /**
+     * Reads and discards a single field, given its tag value.
+     *
+     * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. Otherwise, returns
+     * {@code true}.
+     */
+    public boolean skipField(final int tag) throws IOException {
+        switch (getTagWireType(tag)) {
+            case WIRETYPE_VARINT:
+                readInt32();
+                return true;
+            case WIRETYPE_FIXED64:
+                readRawLittleEndian64();
+                return true;
+            case WIRETYPE_LENGTH_DELIMITED:
+                final int size = readRawVarInt32();
+                if (size < 0) {
+                    throw ProtocolException.negativeSize();
+                }
+                nioBuffer.position(nioBuffer.position() + size);
+                // offset += size;
+                return true;
+            case WIRETYPE_START_GROUP:
+                skipMessage();
+                checkLastTagWas(makeTag(getTagFieldNumber(tag), WIRETYPE_END_GROUP));
+                return true;
+            case WIRETYPE_END_GROUP:
+                return false;
+            case WIRETYPE_FIXED32:
+                readRawLittleEndian32();
+                return true;
+            default:
+                throw ProtocolException.invalidWireType();
+        }
+    }
+
+    /**
+     * Reads and discards an entire message. This will read either until EOF or until an endgroup tag, whichever comes
+     * first.
+     */
+    public void skipMessage() throws IOException {
+        while (true) {
+            final int tag = readTag();
+            if (tag == 0 || !skipField(tag)) {
+                return;
+            }
+        }
+    }
+
+    @Override
+    public  void handleUnknownField(int fieldNumber, Schema schema) throws IOException {
+        skipField(lastTag);
+    }
+
+    @Override
+    public  int readFieldNumber(Schema schema) throws IOException {
+        if (!nioBuffer.hasRemaining()) {
+            lastTag = 0;
+            return 0;
+        }
+
+        // are we reading packed field?
+        if (isCurrentFieldPacked()) {
+            if (packedLimit < nioBuffer.position()) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            // Return field number while reading packed field
+            return lastTag >>> TAG_TYPE_BITS;
+        }
+
+        packedLimit = 0;
+        final int tag = readRawVarInt32();
+        final int fieldNumber = tag >>> TAG_TYPE_BITS;
+        if (fieldNumber == 0) {
+            if (decodeNestedMessageAsGroup && WIRETYPE_TAIL_DELIMITER == (tag & TAG_TYPE_MASK)) {
+                // protostuff's tail delimiter for streaming
+                // 2 options: length-delimited or tail-delimited.
+                lastTag = 0;
+                return 0;
+            }
+            // If we actually read zero, that's not a valid tag.
+            throw ProtocolException.invalidTag();
+        }
+        if (decodeNestedMessageAsGroup && WIRETYPE_END_GROUP == (tag & TAG_TYPE_MASK)) {
+            lastTag = 0;
+            return 0;
+        }
+
+        lastTag = tag;
+        return fieldNumber;
+    }
+
+    /**
+     * Check if this field have been packed into a length-delimited field. If so, update internal state to reflect that
+     * packed fields are being read.
+     */
+    private void checkIfPackedField() throws IOException {
+        // Do we have the start of a packed field?
+        if (packedLimit == 0 && getTagWireType(lastTag) == WIRETYPE_LENGTH_DELIMITED) {
+            final int length = readRawVarInt32();
+            if (length < 0) {
+                throw ProtocolException.negativeSize();
+            }
+
+            if (nioBuffer.position() + length > nioBuffer.limit()) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            this.packedLimit = nioBuffer.position() + length;
+        }
+    }
+
+    /**
+     * Read a {@code double} field value from the internal buffer.
+     */
+    @Override
+    public double readDouble() throws IOException {
+        checkIfPackedField();
+        return Double.longBitsToDouble(readRawLittleEndian64());
+    }
+
+    /**
+     * Read a {@code float} field value from the internal buffer.
+     */
+    @Override
+    public float readFloat() throws IOException {
+        checkIfPackedField();
+        return Float.intBitsToFloat(readRawLittleEndian32());
+    }
+
+    /**
+     * Read a {@code uint64} field value from the internal buffer.
+     */
+    @Override
+    public long readUInt64() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt64();
+    }
+
+    /**
+     * Read an {@code int64} field value from the internal buffer.
+     */
+    @Override
+    public long readInt64() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt64();
+    }
+
+    /**
+     * Read an {@code int32} field value from the internal buffer.
+     */
+    @Override
+    public int readInt32() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read a {@code fixed64} field value from the internal buffer.
+     */
+    @Override
+    public long readFixed64() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian64();
+    }
+
+    /**
+     * Read a {@code fixed32} field value from the internal buffer.
+     */
+    @Override
+    public int readFixed32() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian32();
+    }
+
+    /**
+     * Read a {@code bool} field value from the internal buffer.
+     */
+    @Override
+    public boolean readBool() throws IOException {
+        checkIfPackedField();
+        return nioBuffer.get() != 0;
+    }
+
+    /**
+     * Read a {@code uint32} field value from the internal buffer.
+     */
+    @Override
+    public int readUInt32() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read an enum field value from the internal buffer. Caller is responsible for converting the numeric value to an
+     * actual enum.
+     */
+    @Override
+    public int readEnum() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read an {@code sfixed32} field value from the internal buffer.
+     */
+    @Override
+    public int readSFixed32() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian32();
+    }
+
+    /**
+     * Read an {@code sfixed64} field value from the internal buffer.
+     */
+    @Override
+    public long readSFixed64() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian64();
+    }
+
+    /**
+     * Read an {@code sint32} field value from the internal buffer.
+     */
+    @Override
+    public int readSInt32() throws IOException {
+        checkIfPackedField();
+        final int n = readRawVarInt32();
+        return (n >>> 1) ^ -(n & 1);
+    }
+
+    /**
+     * Read an {@code sint64} field value from the internal buffer.
+     */
+    @Override
+    public long readSInt64() throws IOException {
+        checkIfPackedField();
+        final long n = readRawVarInt64();
+        return (n >>> 1) ^ -(n & 1);
+    }
+
+    @Override
+    public String readString() throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        final int position = nioBuffer.position();
+        String result;
+        if (nioBuffer.hasArray()) {
+            if (UnsafeUtil.hasUnsafe()) {
+                nioBuffer.position(position + length);
+                result = UnsafeUtf8Util.decodeUtf8(nioBuffer.array(), nioBuffer.arrayOffset() + position, length);
+            } else {
+                nioBuffer.position(position + length);
+                result = StringSerializer.STRING.deser(nioBuffer.array(), nioBuffer.arrayOffset() + position, length);
+            }
+        } else {
+            if (UnsafeUtil.hasUnsafe()) {
+                nioBuffer.position(position + length);
+                result = UnsafeUtf8Util.decodeUtf8Direct(nioBuffer, position, length);
+            } else {
+                final byte[] tmp = new byte[length];
+                nioBuffer.get(tmp);
+                result = StringSerializer.STRING.deser(tmp);
+            }
+        }
+        return result;
+    }
+
+    @SuppressWarnings("all")
+    @Override
+    public ByteString readBytes() throws IOException {
+        try {
+            return (ByteString) byteStringWrapMethod.invoke(null, readByteArray());
+        } catch (Exception e) {
+            ThrowUtil.throwException(e);
+        }
+        return null; // never get here
+    }
+
+    @Override
+    public void readBytes(final ByteBuffer bb) throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        bb.put(nioBuffer);
+    }
+
+    @Override
+    public byte[] readByteArray() throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        final byte[] copy = new byte[length];
+        nioBuffer.get(copy);
+        return copy;
+    }
+
+    @Override
+    public  T mergeObject(T value, final Schema schema) throws IOException {
+        if (decodeNestedMessageAsGroup)
+            return mergeObjectEncodedAsGroup(value, schema);
+
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        ByteBuffer dup = nioBuffer.slice();
+        dup.limit(length);
+
+        // save old limit
+        // final int oldLimit = this.limit;
+
+        // this.limit = offset + length;
+
+        if (value == null) {
+            value = schema.newMessage();
+        }
+        ByteBufferInput nestedInput = new ByteBufferInput(dup, false);
+        schema.mergeFrom(nestedInput, value);
+        if (!schema.isInitialized(value)) {
+            throw new UninitializedMessageException(value, schema);
+        }
+        nestedInput.checkLastTagWas(0);
+        // checkLastTagWas(0);
+
+        // restore old limit
+        // this.limit = oldLimit;
+
+        nioBuffer.position(nioBuffer.position() + length);
+        return value;
+    }
+
+    private  T mergeObjectEncodedAsGroup(T value, final Schema schema) throws IOException {
+        if (value == null) {
+            value = schema.newMessage();
+        }
+        schema.mergeFrom(this, value);
+        if (!schema.isInitialized(value)) {
+            throw new UninitializedMessageException(value, schema);
+        }
+        // handling is in #readFieldNumber
+        checkLastTagWas(0);
+        return value;
+    }
+
+    /**
+     * Reads a var int 32 from the internal byte buffer.
+     */
+    public int readRawVarInt32() throws IOException {
+        byte tmp = nioBuffer.get();
+        if (tmp >= 0) {
+            return tmp;
+        }
+        int result = tmp & 0x7f;
+        if ((tmp = nioBuffer.get()) >= 0) {
+            result |= tmp << 7;
+        } else {
+            result |= (tmp & 0x7f) << 7;
+            if ((tmp = nioBuffer.get()) >= 0) {
+                result |= tmp << 14;
+            } else {
+                result |= (tmp & 0x7f) << 14;
+                if ((tmp = nioBuffer.get()) >= 0) {
+                    result |= tmp << 21;
+                } else {
+                    result |= (tmp & 0x7f) << 21;
+                    result |= (tmp = nioBuffer.get()) << 28;
+                    if (tmp < 0) {
+                        // Discard upper 32 bits.
+                        for (int i = 0; i < 5; i++) {
+                            if (nioBuffer.get() >= 0) {
+                                return result;
+                            }
+                        }
+                        throw ProtocolException.malformedVarInt();
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Reads a var int 64 from the internal byte buffer.
+     */
+    public long readRawVarInt64() throws IOException {
+        int shift = 0;
+        long result = 0;
+        while (shift < 64) {
+            final byte b = nioBuffer.get();
+            result |= (long) (b & 0x7F) << shift;
+            if ((b & 0x80) == 0) {
+                return result;
+            }
+            shift += 7;
+        }
+        throw ProtocolException.malformedVarInt();
+    }
+
+    /**
+     * Read a 32-bit little-endian integer from the internal buffer.
+     */
+    public int readRawLittleEndian32() throws IOException {
+        final byte[] bs = new byte[4];
+        nioBuffer.get(bs);
+
+        return (((int) bs[0] & 0xff)) | (((int) bs[1] & 0xff) << 8) | (((int) bs[2] & 0xff) << 16)
+               | (((int) bs[3] & 0xff) << 24);
+    }
+
+    /**
+     * Read a 64-bit little-endian integer from the internal byte buffer.
+     */
+    public long readRawLittleEndian64() throws IOException {
+        final byte[] bs = new byte[8];
+        nioBuffer.get(bs);
+
+        return (((long) bs[0] & 0xff)) | (((long) bs[1] & 0xff) << 8) | (((long) bs[2] & 0xff) << 16)
+               | (((long) bs[3] & 0xff) << 24) | (((long) bs[4] & 0xff) << 32) | (((long) bs[5] & 0xff) << 40)
+               | (((long) bs[6] & 0xff) << 48) | (((long) bs[7] & 0xff) << 56);
+    }
+
+    @Override
+    public void transferByteRangeTo(Output output, boolean utf8String, int fieldNumber, boolean repeated)
+                                                                                                         throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (utf8String) {
+            // if it is a UTF string, we have to call the writeByteRange.
+
+            if (nioBuffer.hasArray()) {
+                output.writeByteRange(true, fieldNumber, nioBuffer.array(),
+                    nioBuffer.arrayOffset() + nioBuffer.position(), length, repeated);
+                nioBuffer.position(nioBuffer.position() + length);
+            } else {
+                byte[] bytes = new byte[length];
+                nioBuffer.get(bytes);
+                output.writeByteRange(true, fieldNumber, bytes, 0, bytes.length, repeated);
+            }
+        } else {
+            // Do the potentially vastly more efficient potential splice call.
+            if (nioBuffer.remaining() < length) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            ByteBuffer dup = nioBuffer.slice();
+            dup.limit(length);
+
+            output.writeBytes(fieldNumber, dup, repeated);
+
+            nioBuffer.position(nioBuffer.position() + length);
+        }
+    }
+
+    /**
+     * Reads a byte array/ByteBuffer value.
+     */
+    @Override
+    public ByteBuffer readByteBuffer() throws IOException {
+        return ByteBuffer.wrap(readByteArray());
+    }
+
+    static {
+        try {
+            byteStringWrapMethod = ByteString.class.getDeclaredMethod("wrap", byte[].class);
+            byteStringWrapMethod.setAccessible(true);
+        } catch (NoSuchMethodException e) {
+            throw new Error(e);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufOutput.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufOutput.java
new file mode 100644
index 0000000..c668aec
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/NioBufOutput.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import io.protostuff.ByteString;
+import io.protostuff.IntSerializer;
+import io.protostuff.Output;
+import io.protostuff.Schema;
+
+import com.alipay.sofa.jraft.rhea.serialization.io.OutputBuf;
+import com.alipay.sofa.jraft.rhea.util.VarInts;
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtf8Util;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+import static io.protostuff.ProtobufOutput.encodeZigZag32;
+import static io.protostuff.ProtobufOutput.encodeZigZag64;
+import static io.protostuff.WireFormat.WIRETYPE_END_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED32;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED64;
+import static io.protostuff.WireFormat.WIRETYPE_LENGTH_DELIMITED;
+import static io.protostuff.WireFormat.WIRETYPE_START_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_VARINT;
+import static io.protostuff.WireFormat.makeTag;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+class NioBufOutput implements Output {
+
+    private static final ReferenceFieldUpdater byteStringBytesGetter = Updaters
+                                                                                             .newReferenceFieldUpdater(
+                                                                                                 ByteString.class,
+                                                                                                 "bytes");
+
+    protected final OutputBuf                                      outputBuf;
+    protected final int                                            maxCapacity;
+    protected ByteBuffer                                           nioBuffer;
+    protected int                                                  capacity;
+
+    NioBufOutput(OutputBuf outputBuf, int minWritableBytes, int maxCapacity) {
+        this.outputBuf = outputBuf;
+        this.maxCapacity = maxCapacity;
+        nioBuffer = outputBuf.nioByteBuffer(minWritableBytes);
+        capacity = nioBuffer.remaining();
+    }
+
+    @Override
+    public void writeInt32(int fieldNumber, int value, boolean repeated) throws IOException {
+        if (value < 0) {
+            writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+            writeVarInt64(value);
+        } else {
+            writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+            writeVarInt32(value);
+        }
+    }
+
+    @Override
+    public void writeUInt32(int fieldNumber, int value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeVarInt32(value);
+    }
+
+    @Override
+    public void writeSInt32(int fieldNumber, int value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeVarInt32(encodeZigZag32(value));
+    }
+
+    @Override
+    public void writeFixed32(int fieldNumber, int value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED32));
+        writeInt32LE(value);
+    }
+
+    @Override
+    public void writeSFixed32(int fieldNumber, int value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED32));
+        writeInt32LE(value);
+    }
+
+    @Override
+    public void writeInt64(int fieldNumber, long value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeVarInt64(value);
+    }
+
+    @Override
+    public void writeUInt64(int fieldNumber, long value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeVarInt64(value);
+    }
+
+    @Override
+    public void writeSInt64(int fieldNumber, long value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeVarInt64(encodeZigZag64(value));
+    }
+
+    @Override
+    public void writeFixed64(int fieldNumber, long value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED64));
+        writeInt64LE(value);
+    }
+
+    @Override
+    public void writeSFixed64(int fieldNumber, long value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED64));
+        writeInt64LE(value);
+    }
+
+    @Override
+    public void writeFloat(int fieldNumber, float value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED32));
+        writeInt32LE(Float.floatToRawIntBits(value));
+    }
+
+    @Override
+    public void writeDouble(int fieldNumber, double value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_FIXED64));
+        writeInt64LE(Double.doubleToRawLongBits(value));
+    }
+
+    @Override
+    public void writeBool(int fieldNumber, boolean value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_VARINT));
+        writeByte(value ? (byte) 0x01 : 0x00);
+    }
+
+    @Override
+    public void writeEnum(int fieldNumber, int value, boolean repeated) throws IOException {
+        writeInt32(fieldNumber, value, repeated);
+    }
+
+    @Override
+    public void writeString(int fieldNumber, CharSequence value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED));
+
+        // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()),
+        // and at most 3 times of it. We take advantage of this in both branches below.
+        int minLength = value.length();
+        int maxLength = minLength * UnsafeUtf8Util.MAX_BYTES_PER_CHAR;
+        int minLengthVarIntSize = VarInts.computeRawVarInt32Size(minLength);
+        int maxLengthVarIntSize = VarInts.computeRawVarInt32Size(maxLength);
+        if (minLengthVarIntSize == maxLengthVarIntSize) {
+            int position = nioBuffer.position();
+
+            ensureCapacity(maxLengthVarIntSize + maxLength);
+
+            // Save the current position and increment past the length field. We'll come back
+            // and write the length field after the encoding is complete.
+            int stringStartPos = position + maxLengthVarIntSize;
+            nioBuffer.position(stringStartPos);
+
+            int length;
+            // Encode the string.
+            if (nioBuffer.isDirect()) {
+                UnsafeUtf8Util.encodeUtf8Direct(value, nioBuffer);
+                // Write the length and advance the position.
+                length = nioBuffer.position() - stringStartPos;
+            } else {
+                int offset = nioBuffer.arrayOffset() + stringStartPos;
+                int outIndex = UnsafeUtf8Util.encodeUtf8(value, nioBuffer.array(), offset, nioBuffer.remaining());
+                length = outIndex - offset;
+            }
+            nioBuffer.position(position);
+            writeVarInt32(length);
+            nioBuffer.position(stringStartPos + length);
+        } else {
+            // Calculate and write the encoded length.
+            int length = UnsafeUtf8Util.encodedLength(value);
+            writeVarInt32(length);
+
+            ensureCapacity(length);
+
+            if (nioBuffer.isDirect()) {
+                // Write the string and advance the position.
+                UnsafeUtf8Util.encodeUtf8Direct(value, nioBuffer);
+            } else {
+                int pos = nioBuffer.position();
+                UnsafeUtf8Util.encodeUtf8(value, nioBuffer.array(), nioBuffer.arrayOffset() + pos,
+                    nioBuffer.remaining());
+                nioBuffer.position(pos + length);
+            }
+        }
+    }
+
+    @Override
+    public void writeBytes(int fieldNumber, ByteString value, boolean repeated) throws IOException {
+        writeByteArray(fieldNumber, byteStringBytesGetter.get(value), repeated);
+    }
+
+    @Override
+    public void writeByteArray(int fieldNumber, byte[] value, boolean repeated) throws IOException {
+        writeByteRange(false, fieldNumber, value, 0, value.length, repeated);
+    }
+
+    @Override
+    public void writeByteRange(boolean utf8String, int fieldNumber, byte[] value, int offset, int length,
+                               boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED));
+        writeVarInt32(length);
+        writeByteArray(value, offset, length);
+    }
+
+    @Override
+    public  void writeObject(int fieldNumber, T value, Schema schema, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_START_GROUP));
+        schema.writeTo(this, value);
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_END_GROUP));
+    }
+
+    @Override
+    public void writeBytes(int fieldNumber, ByteBuffer value, boolean repeated) throws IOException {
+        writeByteRange(false, fieldNumber, value.array(), value.arrayOffset() + value.position(), value.remaining(),
+            repeated);
+    }
+
+    protected void writeVarInt32(int value) throws IOException {
+        ensureCapacity(5);
+        while (true) {
+            if ((value & ~0x7F) == 0) {
+                nioBuffer.put((byte) value);
+                return;
+            } else {
+                nioBuffer.put((byte) ((value & 0x7F) | 0x80));
+                value >>>= 7;
+            }
+        }
+    }
+
+    protected void writeVarInt64(long value) throws IOException {
+        ensureCapacity(10);
+        while (true) {
+            if ((value & ~0x7FL) == 0) {
+                nioBuffer.put((byte) value);
+                return;
+            } else {
+                nioBuffer.put((byte) (((int) value & 0x7F) | 0x80));
+                value >>>= 7;
+            }
+        }
+    }
+
+    protected void writeInt32LE(final int value) throws IOException {
+        ensureCapacity(4);
+        IntSerializer.writeInt32LE(value, nioBuffer);
+    }
+
+    protected void writeInt64LE(final long value) throws IOException {
+        ensureCapacity(8);
+        IntSerializer.writeInt64LE(value, nioBuffer);
+    }
+
+    protected void writeByte(final byte value) throws IOException {
+        ensureCapacity(1);
+        nioBuffer.put(value);
+    }
+
+    protected void writeByteArray(final byte[] value, final int offset, final int length) throws IOException {
+        ensureCapacity(length);
+        nioBuffer.put(value, offset, length);
+    }
+
+    protected void ensureCapacity(int required) throws ProtocolException {
+        if (nioBuffer.remaining() < required) {
+            int position = nioBuffer.position();
+
+            while (capacity - position < required) {
+                if (capacity == maxCapacity) {
+                    throw new ProtocolException("Buffer overflow. Available: " + (capacity - position) + ", required: "
+                                                + required);
+                }
+                capacity = Math.min(capacity << 1, maxCapacity);
+                if (capacity < 0) {
+                    capacity = maxCapacity;
+                }
+            }
+
+            nioBuffer = outputBuf.nioByteBuffer(capacity - position);
+            capacity = nioBuffer.limit();
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Outputs.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Outputs.java
new file mode 100644
index 0000000..fa5bdec
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/Outputs.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import io.protostuff.LinkedBuffer;
+import io.protostuff.Output;
+import io.protostuff.ProtostuffOutput;
+import io.protostuff.WriteSession;
+
+import com.alipay.sofa.jraft.rhea.serialization.io.OutputBuf;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class Outputs {
+
+    public static Output getOutput(final OutputBuf outputBuf) {
+        if (UnsafeUtil.hasUnsafe() && outputBuf.hasMemoryAddress()) {
+            return new UnsafeNioBufOutput(outputBuf, -1, Integer.MAX_VALUE);
+        }
+        return new NioBufOutput(outputBuf, -1, Integer.MAX_VALUE);
+    }
+
+    public static Output getOutput(final LinkedBuffer buf) {
+        return new ProtostuffOutput(buf);
+    }
+
+    public static byte[] toByteArray(final Output output) {
+        if (output instanceof WriteSession) {
+            return ((WriteSession) output).toByteArray();
+        }
+        throw new IllegalArgumentException("Output [" + output.getClass().getName()
+                                           + "] must be a WriteSession's implementation");
+    }
+
+    private Outputs() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/ProtocolException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/ProtocolException.java
new file mode 100644
index 0000000..aff4bad
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/ProtocolException.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import io.protostuff.ProtobufException;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ProtocolException extends ProtobufException {
+
+    private static final long serialVersionUID = -1L;
+
+    public ProtocolException(String description) {
+        super(description);
+    }
+
+    public ProtocolException(String description, Throwable cause) {
+        super(description, cause);
+    }
+
+    static ProtocolException misreportedSize() {
+        return new ProtocolException("CodedInput encountered an embedded string or bytes "
+                                     + "that misreported its size.");
+    }
+
+    static ProtocolException negativeSize() {
+        return new ProtocolException("CodedInput encountered an embedded string or message "
+                                     + "which claimed to have negative size.");
+    }
+
+    static ProtocolException malformedVarInt() {
+        return new ProtocolException("CodedInput encountered a malformed varint.");
+    }
+
+    static ProtocolException invalidTag() {
+        return new ProtocolException("Protocol message contained an invalid tag (zero).");
+    }
+
+    static ProtocolException invalidEndTag() {
+        return new ProtocolException("Protocol message end-group tag did not match expected tag.");
+    }
+
+    static ProtocolException invalidWireType() {
+        return new ProtocolException("Protocol message tag had invalid wire type.");
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufInput.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufInput.java
new file mode 100644
index 0000000..543a73c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufInput.java
@@ -0,0 +1,635 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.ByteBuffer;
+
+import io.protostuff.ByteBufferInput;
+import io.protostuff.ByteString;
+import io.protostuff.Input;
+import io.protostuff.Output;
+import io.protostuff.ProtobufException;
+import io.protostuff.Schema;
+import io.protostuff.UninitializedMessageException;
+
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.rhea.util.internal.UnsafeDirectBufferUtil;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtf8Util;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+import static io.protostuff.WireFormat.WIRETYPE_END_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED32;
+import static io.protostuff.WireFormat.WIRETYPE_FIXED64;
+import static io.protostuff.WireFormat.WIRETYPE_LENGTH_DELIMITED;
+import static io.protostuff.WireFormat.WIRETYPE_START_GROUP;
+import static io.protostuff.WireFormat.WIRETYPE_TAIL_DELIMITER;
+import static io.protostuff.WireFormat.WIRETYPE_VARINT;
+import static io.protostuff.WireFormat.getTagFieldNumber;
+import static io.protostuff.WireFormat.getTagWireType;
+import static io.protostuff.WireFormat.makeTag;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+class UnsafeNioBufInput implements Input {
+
+    static final int         TAG_TYPE_BITS = 3;
+    static final int         TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1;
+
+    static final Method      byteStringWrapMethod;
+
+    private final ByteBuffer nioBuffer;
+    private int              lastTag       = 0;
+    private int              packedLimit   = 0;
+
+    /**
+     * Start address of the memory buffer The memory buffer should be non-movable, which normally means that is is allocated
+     * off-heap
+     */
+    private long             memoryAddress;
+
+    /**
+     * If true, the nested messages are group-encoded
+     */
+    public final boolean     decodeNestedMessageAsGroup;
+
+    /**
+     * An input for a ByteBuffer
+     *
+     * @param nioBuffer         the buffer to read from, it will not be sliced
+     * @param protostuffMessage if we are parsing a protostuff (true) or protobuf (false) message
+     */
+    UnsafeNioBufInput(ByteBuffer nioBuffer, boolean protostuffMessage) {
+        this.nioBuffer = nioBuffer;
+        this.decodeNestedMessageAsGroup = protostuffMessage;
+        updateBufferAddress();
+    }
+
+    /**
+     * Returns the current offset (the position).
+     */
+    public int currentOffset() {
+        return nioBuffer.position();
+    }
+
+    /**
+     * Returns the current limit (the end index).
+     */
+    public int currentLimit() {
+        return nioBuffer.limit();
+    }
+
+    /**
+     * Return true if currently reading packed field
+     */
+    public boolean isCurrentFieldPacked() {
+        return packedLimit != 0 && packedLimit != nioBuffer.position();
+    }
+
+    /**
+     * Returns the last tag.
+     */
+    public int getLastTag() {
+        return lastTag;
+    }
+
+    /**
+     * Attempt to read a field tag, returning zero if we have reached EOF. Protocol message parsers use this to read
+     * tags, since a protocol message may legally end wherever a tag occurs, and zero is not a valid tag number.
+     */
+    public int readTag() throws IOException {
+        if (!nioBuffer.hasRemaining()) {
+            lastTag = 0;
+            return 0;
+        }
+
+        final int tag = readRawVarInt32();
+        if (tag >>> TAG_TYPE_BITS == 0) {
+            // If we actually read zero, that's not a valid tag.
+            throw ProtocolException.invalidTag();
+        }
+        lastTag = tag;
+        return tag;
+    }
+
+    /**
+     * Verifies that the last call to readTag() returned the given tag value. This is used to verify that a nested group
+     * ended with the correct end tag.
+     *
+     * @throws ProtobufException {@code value} does not match the last tag.
+     */
+    public void checkLastTagWas(final int value) throws ProtobufException {
+        if (lastTag != value) {
+            throw ProtocolException.invalidEndTag();
+        }
+    }
+
+    /**
+     * Reads and discards a single field, given its tag value.
+     *
+     * @return {@code false} if the tag is an endgroup tag, in which case nothing is skipped. Otherwise, returns
+     * {@code true}.
+     */
+    public boolean skipField(final int tag) throws IOException {
+        switch (getTagWireType(tag)) {
+            case WIRETYPE_VARINT:
+                readInt32();
+                return true;
+            case WIRETYPE_FIXED64:
+                readRawLittleEndian64();
+                return true;
+            case WIRETYPE_LENGTH_DELIMITED:
+                final int size = readRawVarInt32();
+                if (size < 0) {
+                    throw ProtocolException.negativeSize();
+                }
+                nioBuffer.position(nioBuffer.position() + size);
+                // offset += size;
+                return true;
+            case WIRETYPE_START_GROUP:
+                skipMessage();
+                checkLastTagWas(makeTag(getTagFieldNumber(tag), WIRETYPE_END_GROUP));
+                return true;
+            case WIRETYPE_END_GROUP:
+                return false;
+            case WIRETYPE_FIXED32:
+                readRawLittleEndian32();
+                return true;
+            default:
+                throw ProtocolException.invalidWireType();
+        }
+    }
+
+    /**
+     * Reads and discards an entire message. This will read either until EOF or until an endgroup tag, whichever comes
+     * first.
+     */
+    public void skipMessage() throws IOException {
+        while (true) {
+            final int tag = readTag();
+            if (tag == 0 || !skipField(tag)) {
+                return;
+            }
+        }
+    }
+
+    @Override
+    public  void handleUnknownField(int fieldNumber, Schema schema) throws IOException {
+        skipField(lastTag);
+    }
+
+    @Override
+    public  int readFieldNumber(Schema schema) throws IOException {
+        if (!nioBuffer.hasRemaining()) {
+            lastTag = 0;
+            return 0;
+        }
+
+        // are we reading packed field?
+        if (isCurrentFieldPacked()) {
+            if (packedLimit < nioBuffer.position()) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            // Return field number while reading packed field
+            return lastTag >>> TAG_TYPE_BITS;
+        }
+
+        packedLimit = 0;
+        final int tag = readRawVarInt32();
+        final int fieldNumber = tag >>> TAG_TYPE_BITS;
+        if (fieldNumber == 0) {
+            if (decodeNestedMessageAsGroup && WIRETYPE_TAIL_DELIMITER == (tag & TAG_TYPE_MASK)) {
+                // protostuff's tail delimiter for streaming
+                // 2 options: length-delimited or tail-delimited.
+                lastTag = 0;
+                return 0;
+            }
+            // If we actually read zero, that's not a valid tag.
+            throw ProtocolException.invalidTag();
+        }
+        if (decodeNestedMessageAsGroup && WIRETYPE_END_GROUP == (tag & TAG_TYPE_MASK)) {
+            lastTag = 0;
+            return 0;
+        }
+
+        lastTag = tag;
+        return fieldNumber;
+    }
+
+    /**
+     * Check if this field have been packed into a length-delimited field. If so, update internal state to reflect that
+     * packed fields are being read.
+     */
+    private void checkIfPackedField() throws IOException {
+        // Do we have the start of a packed field?
+        if (packedLimit == 0 && getTagWireType(lastTag) == WIRETYPE_LENGTH_DELIMITED) {
+            final int length = readRawVarInt32();
+            if (length < 0) {
+                throw ProtocolException.negativeSize();
+            }
+
+            if (nioBuffer.position() + length > nioBuffer.limit()) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            this.packedLimit = nioBuffer.position() + length;
+        }
+    }
+
+    /**
+     * Read a {@code double} field value from the internal buffer.
+     */
+    @Override
+    public double readDouble() throws IOException {
+        checkIfPackedField();
+        return Double.longBitsToDouble(readRawLittleEndian64());
+    }
+
+    /**
+     * Read a {@code float} field value from the internal buffer.
+     */
+    @Override
+    public float readFloat() throws IOException {
+        checkIfPackedField();
+        return Float.intBitsToFloat(readRawLittleEndian32());
+    }
+
+    /**
+     * Read a {@code uint64} field value from the internal buffer.
+     */
+    @Override
+    public long readUInt64() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt64();
+    }
+
+    /**
+     * Read an {@code int64} field value from the internal buffer.
+     */
+    @Override
+    public long readInt64() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt64();
+    }
+
+    /**
+     * Read an {@code int32} field value from the internal buffer.
+     */
+    @Override
+    public int readInt32() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read a {@code fixed64} field value from the internal buffer.
+     */
+    @Override
+    public long readFixed64() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian64();
+    }
+
+    /**
+     * Read a {@code fixed32} field value from the internal buffer.
+     */
+    @Override
+    public int readFixed32() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian32();
+    }
+
+    /**
+     * Read a {@code bool} field value from the internal buffer.
+     */
+    @Override
+    public boolean readBool() throws IOException {
+        checkIfPackedField();
+        int position = nioBuffer.position();
+        boolean result = UnsafeDirectBufferUtil.getByte(address(position)) != 0;
+        nioBuffer.position(position + 1);
+        return result;
+    }
+
+    /**
+     * Read a {@code uint32} field value from the internal buffer.
+     */
+    @Override
+    public int readUInt32() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read an enum field value from the internal buffer. Caller is responsible for converting the numeric value to an
+     * actual enum.
+     */
+    @Override
+    public int readEnum() throws IOException {
+        checkIfPackedField();
+        return readRawVarInt32();
+    }
+
+    /**
+     * Read an {@code sfixed32} field value from the internal buffer.
+     */
+    @Override
+    public int readSFixed32() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian32();
+    }
+
+    /**
+     * Read an {@code sfixed64} field value from the internal buffer.
+     */
+    @Override
+    public long readSFixed64() throws IOException {
+        checkIfPackedField();
+        return readRawLittleEndian64();
+    }
+
+    /**
+     * Read an {@code sint32} field value from the internal buffer.
+     */
+    @Override
+    public int readSInt32() throws IOException {
+        checkIfPackedField();
+        final int n = readRawVarInt32();
+        return (n >>> 1) ^ -(n & 1);
+    }
+
+    /**
+     * Read an {@code sint64} field value from the internal buffer.
+     */
+    @Override
+    public long readSInt64() throws IOException {
+        checkIfPackedField();
+        final long n = readRawVarInt64();
+        return (n >>> 1) ^ -(n & 1);
+    }
+
+    @Override
+    public String readString() throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        int position = nioBuffer.position();
+        String result = UnsafeUtf8Util.decodeUtf8Direct(nioBuffer, position, length);
+        nioBuffer.position(position + length);
+        return result;
+    }
+
+    @SuppressWarnings("all")
+    @Override
+    public ByteString readBytes() throws IOException {
+        try {
+            return (ByteString) byteStringWrapMethod.invoke(null, readByteArray());
+        } catch (Exception e) {
+            ThrowUtil.throwException(e);
+        }
+        return null; // never get here
+    }
+
+    @Override
+    public void readBytes(final ByteBuffer bb) throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        bb.put(nioBuffer);
+    }
+
+    @Override
+    public byte[] readByteArray() throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        final byte[] copy = new byte[length];
+        int position = nioBuffer.position();
+        UnsafeDirectBufferUtil.getBytes(address(position), copy, 0, length);
+        nioBuffer.position(position + length);
+        return copy;
+    }
+
+    @Override
+    public  T mergeObject(T value, final Schema schema) throws IOException {
+        if (decodeNestedMessageAsGroup) {
+            return mergeObjectEncodedAsGroup(value, schema);
+        }
+
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (nioBuffer.remaining() < length) {
+            throw ProtocolException.misreportedSize();
+        }
+
+        ByteBuffer dup = nioBuffer.slice();
+        dup.limit(length);
+
+        if (value == null) {
+            value = schema.newMessage();
+        }
+        ByteBufferInput nestedInput = new ByteBufferInput(dup, false);
+        schema.mergeFrom(nestedInput, value);
+        if (!schema.isInitialized(value)) {
+            throw new UninitializedMessageException(value, schema);
+        }
+        nestedInput.checkLastTagWas(0);
+
+        nioBuffer.position(nioBuffer.position() + length);
+        return value;
+    }
+
+    private  T mergeObjectEncodedAsGroup(T value, final Schema schema) throws IOException {
+        if (value == null) {
+            value = schema.newMessage();
+        }
+        schema.mergeFrom(this, value);
+        if (!schema.isInitialized(value)) {
+            throw new UninitializedMessageException(value, schema);
+        }
+        // handling is in #readFieldNumber
+        checkLastTagWas(0);
+        return value;
+    }
+
+    /**
+     * Reads a var int 32 from the internal byte buffer.
+     */
+    public int readRawVarInt32() throws IOException {
+        int position = nioBuffer.position();
+        byte tmp = UnsafeDirectBufferUtil.getByte(address(position++));
+        if (tmp >= 0) {
+            nioBuffer.position(position);
+            return tmp;
+        }
+        int result = tmp & 0x7f;
+        if ((tmp = UnsafeDirectBufferUtil.getByte(address(position++))) >= 0) {
+            result |= tmp << 7;
+        } else {
+            result |= (tmp & 0x7f) << 7;
+            if ((tmp = UnsafeDirectBufferUtil.getByte(address(position++))) >= 0) {
+                result |= tmp << 14;
+            } else {
+                result |= (tmp & 0x7f) << 14;
+                if ((tmp = UnsafeDirectBufferUtil.getByte(address(position++))) >= 0) {
+                    result |= tmp << 21;
+                } else {
+                    result |= (tmp & 0x7f) << 21;
+                    result |= (tmp = UnsafeDirectBufferUtil.getByte(address(position++))) << 28;
+                    if (tmp < 0) {
+                        // Discard upper 32 bits.
+                        for (int i = 0; i < 5; i++) {
+                            if (UnsafeDirectBufferUtil.getByte(address(position++)) >= 0) {
+                                nioBuffer.position(position);
+                                return result;
+                            }
+                        }
+                        throw ProtocolException.malformedVarInt();
+                    }
+                }
+            }
+        }
+        nioBuffer.position(position);
+        return result;
+    }
+
+    /**
+     * Reads a var int 64 from the internal byte buffer.
+     */
+    public long readRawVarInt64() throws IOException {
+        int shift = 0;
+        long result = 0;
+        int position = nioBuffer.position();
+        while (shift < 64) {
+            final byte b = UnsafeDirectBufferUtil.getByte(address(position++));
+            result |= (long) (b & 0x7F) << shift;
+            if ((b & 0x80) == 0) {
+                nioBuffer.position(position);
+                return result;
+            }
+            shift += 7;
+        }
+        throw ProtocolException.malformedVarInt();
+    }
+
+    /**
+     * Read a 32-bit little-endian integer from the internal buffer.
+     */
+    public int readRawLittleEndian32() throws IOException {
+        int position = nioBuffer.position();
+        int result = UnsafeDirectBufferUtil.getIntLE(address(position));
+        nioBuffer.position(position + 4);
+        return result;
+    }
+
+    /**
+     * Read a 64-bit little-endian integer from the internal byte buffer.
+     */
+    public long readRawLittleEndian64() throws IOException {
+        int position = nioBuffer.position();
+        long result = UnsafeDirectBufferUtil.getLongLE(address(position));
+        nioBuffer.position(position + 8);
+        return result;
+    }
+
+    @Override
+    public void transferByteRangeTo(Output output, boolean utf8String, int fieldNumber, boolean repeated)
+                                                                                                         throws IOException {
+        final int length = readRawVarInt32();
+        if (length < 0) {
+            throw ProtocolException.negativeSize();
+        }
+
+        if (utf8String) {
+            // if it is a UTF string, we have to call the writeByteRange.
+
+            if (nioBuffer.hasArray()) {
+                output.writeByteRange(true, fieldNumber, nioBuffer.array(),
+                    nioBuffer.arrayOffset() + nioBuffer.position(), length, repeated);
+                nioBuffer.position(nioBuffer.position() + length);
+            } else {
+                byte[] bytes = new byte[length];
+                int position = nioBuffer.position();
+                UnsafeDirectBufferUtil.getBytes(address(position), bytes, 0, length);
+                nioBuffer.position(position + length);
+                output.writeByteRange(true, fieldNumber, bytes, 0, bytes.length, repeated);
+            }
+        } else {
+            // Do the potentially vastly more efficient potential splice call.
+            if (nioBuffer.remaining() < length) {
+                throw ProtocolException.misreportedSize();
+            }
+
+            ByteBuffer dup = nioBuffer.slice();
+            dup.limit(length);
+
+            output.writeBytes(fieldNumber, dup, repeated);
+
+            nioBuffer.position(nioBuffer.position() + length);
+        }
+    }
+
+    /**
+     * Reads a byte array/ByteBuffer value.
+     */
+    @Override
+    public ByteBuffer readByteBuffer() throws IOException {
+        return ByteBuffer.wrap(readByteArray());
+    }
+
+    private long address(int position) {
+        return memoryAddress + position;
+    }
+
+    private void updateBufferAddress() {
+        memoryAddress = UnsafeUtil.addressOffset(nioBuffer);
+    }
+
+    static {
+        try {
+            byteStringWrapMethod = ByteString.class.getDeclaredMethod("wrap", byte[].class);
+            byteStringWrapMethod.setAccessible(true);
+        } catch (NoSuchMethodException e) {
+            throw new Error(e);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufOutput.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufOutput.java
new file mode 100644
index 0000000..572665d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/impl/protostuff/io/UnsafeNioBufOutput.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.impl.protostuff.io;
+
+import java.io.IOException;
+
+import com.alipay.sofa.jraft.rhea.serialization.io.OutputBuf;
+import com.alipay.sofa.jraft.rhea.util.VarInts;
+import com.alipay.sofa.jraft.rhea.util.internal.UnsafeDirectBufferUtil;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtf8Util;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+import static io.protostuff.WireFormat.WIRETYPE_LENGTH_DELIMITED;
+import static io.protostuff.WireFormat.makeTag;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+class UnsafeNioBufOutput extends NioBufOutput {
+
+    /**
+     * Start address of the memory buffer The memory buffer should be non-movable, which normally means that is is allocated
+     * off-heap
+     */
+    private long memoryAddress;
+
+    UnsafeNioBufOutput(OutputBuf outputBuf, int minWritableBytes, int maxCapacity) {
+        super(outputBuf, minWritableBytes, maxCapacity);
+        updateBufferAddress();
+    }
+
+    @Override
+    public void writeString(int fieldNumber, CharSequence value, boolean repeated) throws IOException {
+        writeVarInt32(makeTag(fieldNumber, WIRETYPE_LENGTH_DELIMITED));
+
+        // UTF-8 byte length of the string is at least its UTF-16 code unit length (value.length()),
+        // and at most 3 times of it. We take advantage of this in both branches below.
+        int minLength = value.length();
+        int maxLength = minLength * UnsafeUtf8Util.MAX_BYTES_PER_CHAR;
+        int minLengthVarIntSize = VarInts.computeRawVarInt32Size(minLength);
+        int maxLengthVarIntSize = VarInts.computeRawVarInt32Size(maxLength);
+        if (minLengthVarIntSize == maxLengthVarIntSize) {
+            int position = nioBuffer.position();
+
+            ensureCapacity(maxLengthVarIntSize + maxLength);
+
+            // Save the current position and increment past the length field. We'll come back
+            // and write the length field after the encoding is complete.
+            int stringStartPos = position + maxLengthVarIntSize;
+            nioBuffer.position(stringStartPos);
+
+            // Encode the string.
+            UnsafeUtf8Util.encodeUtf8Direct(value, nioBuffer);
+
+            // Write the length and advance the position.
+            int length = nioBuffer.position() - stringStartPos;
+            nioBuffer.position(position);
+            writeVarInt32(length);
+            nioBuffer.position(stringStartPos + length);
+        } else {
+            // Calculate and write the encoded length.
+            int length = UnsafeUtf8Util.encodedLength(value);
+            writeVarInt32(length);
+
+            ensureCapacity(length);
+
+            // Write the string and advance the position.
+            UnsafeUtf8Util.encodeUtf8Direct(value, nioBuffer);
+        }
+    }
+
+    @Override
+    protected void writeVarInt32(int value) throws IOException {
+        //        ensureCapacity(5);
+        //        int position = nioBuffer.position();
+        //        while (true) {
+        //            if ((value & ~0x7F) == 0) {
+        //                UnsafeDirectBufferUtil.setByte(address(position++), (byte) value);
+        //                nioBuffer.position(position);
+        //                return;
+        //            } else {
+        //                UnsafeDirectBufferUtil.setByte(address(position++), (byte) ((value & 0x7F) | 0x80));
+        //                value >>>= 7;
+        //            }
+        //        }
+        //
+        // The following implementation is no different from the above code function,
+        // just aggregate multiple setBytes into a setShort/setInt
+        //
+        ensureCapacity(5);
+        int position = nioBuffer.position();
+        if ((value & (~0 << 7)) == 0) {
+            // size == 1
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) value);
+        } else if ((value & (~0 << 14)) == 0) {
+            // size == 2
+            UnsafeDirectBufferUtil.setShort(address(position), (((value & 0x7F) | 0x80) << 8) | (value >>> 7));
+            position += 2;
+        } else if ((value & (~0 << 21)) == 0) {
+            // size == 3
+            UnsafeDirectBufferUtil.setShort(address(position), (((value & 0x7F) | 0x80) << 8)
+                                                               | ((value >>> 7 & 0x7F) | 0x80));
+            position += 2;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 14));
+        } else if ((value & (~0 << 28)) == 0) {
+            // size == 4
+            UnsafeDirectBufferUtil.setInt(address(position), (((value & 0x7F) | 0x80) << 24)
+                                                             | (((value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | (((value >>> 14 & 0x7F) | 0x80) << 8) | (value >>> 21));
+            position += 4;
+        } else {
+            // size == 5
+            UnsafeDirectBufferUtil.setInt(address(position), (((value & 0x7F) | 0x80) << 24)
+                                                             | (((value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | (((value >>> 14 & 0x7F) | 0x80) << 8)
+                                                             | ((value >>> 21 & 0x7F) | 0x80));
+            position += 4;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 28));
+        }
+        nioBuffer.position(position);
+    }
+
+    @Override
+    protected void writeVarInt64(long value) throws IOException {
+        //        ensureCapacity(10);
+        //        int position = nioBuffer.position();
+        //        while (true) {
+        //            if ((value & ~0x7FL) == 0) {
+        //                UnsafeDirectBufferUtil.setByte(address(position++), (byte) value);
+        //                nioBuffer.position(position);
+        //                return;
+        //            } else {
+        //                UnsafeDirectBufferUtil.setByte(address(position++), (byte) (((int) value & 0x7F) | 0x80));
+        //                value >>>= 7;
+        //            }
+        //        }
+        //
+        // The following implementation is no different from the above code function,
+        // just aggregate multiple setBytes into a setShort/setInt
+        //
+        ensureCapacity(10);
+        int position = nioBuffer.position();
+        // Handle two popular special cases up front ...
+        if ((value & (~0L << 7)) == 0) {
+            // size == 1
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) value);
+        } else if (value < 0L) {
+            // size == 10
+            UnsafeDirectBufferUtil.setLong(address(position), (((value & 0x7F) | 0x80) << 56)
+                                                              | (((value >>> 7 & 0x7F) | 0x80) << 48)
+                                                              | (((value >>> 14 & 0x7F) | 0x80) << 40)
+                                                              | (((value >>> 21 & 0x7F) | 0x80) << 32)
+                                                              | (((value >>> 28 & 0x7F) | 0x80) << 24)
+                                                              | (((value >>> 35 & 0x7F) | 0x80) << 16)
+                                                              | (((value >>> 42 & 0x7F) | 0x80) << 8)
+                                                              | ((value >>> 49 & 0x7F) | 0x80));
+            position += 8;
+            UnsafeDirectBufferUtil.setShort(address(position), ((((int) (value >>> 56) & 0x7F) | 0x80) << 8)
+                                                               | (int) (value >>> 63));
+            position += 2;
+        }
+        // ... leaving us with 8 remaining [2, 3, 4, 5, 6, 7, 8, 9]
+        else if ((value & (~0L << 14)) == 0) {
+            // size == 2
+            UnsafeDirectBufferUtil.setShort(address(position), ((((int) value & 0x7F) | 0x80) << 8)
+                                                               | (byte) (value >>> 7));
+            position += 2;
+        } else if ((value & (~0L << 21)) == 0) {
+            // size == 3
+            UnsafeDirectBufferUtil.setShort(address(position), ((((int) value & 0x7F) | 0x80) << 8)
+                                                               | (((int) value >>> 7 & 0x7F) | 0x80));
+            position += 2;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 14));
+        } else if ((value & (~0L << 28)) == 0) {
+            // size == 4
+            UnsafeDirectBufferUtil.setInt(address(position), ((((int) value & 0x7F) | 0x80) << 24)
+                                                             | ((((int) value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | ((((int) value >>> 14 & 0x7F) | 0x80) << 8)
+                                                             | ((int) (value >>> 21)));
+            position += 4;
+        } else if ((value & (~0L << 35)) == 0) {
+            // size == 5
+            UnsafeDirectBufferUtil.setInt(address(position), ((((int) value & 0x7F) | 0x80) << 24)
+                                                             | ((((int) value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | ((((int) value >>> 14 & 0x7F) | 0x80) << 8)
+                                                             | (((int) value >>> 21 & 0x7F) | 0x80));
+            position += 4;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 28));
+        } else if ((value & (~0L << 42)) == 0) {
+            // size == 6
+            UnsafeDirectBufferUtil.setInt(address(position), ((((int) value & 0x7F) | 0x80) << 24)
+                                                             | ((((int) value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | ((((int) value >>> 14 & 0x7F) | 0x80) << 8)
+                                                             | (((int) value >>> 21 & 0x7F) | 0x80));
+            position += 4;
+            UnsafeDirectBufferUtil.setShort(address(position), ((((int) (value >>> 28) & 0x7F) | 0x80) << 8)
+                                                               | (int) (value >>> 35));
+            position += 2;
+        } else if ((value & (~0L << 49)) == 0) {
+            // size == 7
+            UnsafeDirectBufferUtil.setInt(address(position), ((((int) value & 0x7F) | 0x80) << 24)
+                                                             | ((((int) value >>> 7 & 0x7F) | 0x80) << 16)
+                                                             | ((((int) value >>> 14 & 0x7F) | 0x80) << 8)
+                                                             | (((int) value >>> 21 & 0x7F) | 0x80));
+            position += 4;
+            UnsafeDirectBufferUtil.setShort(address(position), ((((int) (value >>> 28) & 0x7F) | 0x80) << 8)
+                                                               | (((int) (value >>> 35) & 0x7F) | 0x80));
+            position += 2;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 42));
+        } else if ((value & (~0L << 56)) == 0) {
+            // size == 8
+            UnsafeDirectBufferUtil.setLong(address(position), (((value & 0x7F) | 0x80) << 56)
+                                                              | (((value >>> 7 & 0x7F) | 0x80) << 48)
+                                                              | (((value >>> 14 & 0x7F) | 0x80) << 40)
+                                                              | (((value >>> 21 & 0x7F) | 0x80) << 32)
+                                                              | (((value >>> 28 & 0x7F) | 0x80) << 24)
+                                                              | (((value >>> 35 & 0x7F) | 0x80) << 16)
+                                                              | (((value >>> 42 & 0x7F) | 0x80) << 8) | (value >>> 49));
+            position += 8;
+        } else {
+            // size == 9 (value & (~0L << 63)) == 0
+            UnsafeDirectBufferUtil.setLong(address(position), (((value & 0x7F) | 0x80) << 56)
+                                                              | (((value >>> 7 & 0x7F) | 0x80) << 48)
+                                                              | (((value >>> 14 & 0x7F) | 0x80) << 40)
+                                                              | (((value >>> 21 & 0x7F) | 0x80) << 32)
+                                                              | (((value >>> 28 & 0x7F) | 0x80) << 24)
+                                                              | (((value >>> 35 & 0x7F) | 0x80) << 16)
+                                                              | (((value >>> 42 & 0x7F) | 0x80) << 8)
+                                                              | ((value >>> 49 & 0x7F) | 0x80));
+            position += 8;
+            UnsafeDirectBufferUtil.setByte(address(position++), (byte) (value >>> 56));
+        }
+        nioBuffer.position(position);
+    }
+
+    @Override
+    protected void writeInt32LE(int value) throws IOException {
+        ensureCapacity(4);
+        int position = nioBuffer.position();
+        UnsafeDirectBufferUtil.setIntLE(address(position), value);
+        nioBuffer.position(position + 4);
+    }
+
+    @Override
+    protected void writeInt64LE(long value) throws IOException {
+        ensureCapacity(8);
+        int position = nioBuffer.position();
+        UnsafeDirectBufferUtil.setLongLE(address(position), value);
+        nioBuffer.position(position + 8);
+    }
+
+    @Override
+    protected void writeByte(byte value) throws IOException {
+        ensureCapacity(1);
+        int position = nioBuffer.position();
+        UnsafeDirectBufferUtil.setByte(address(position), value);
+        nioBuffer.position(position + 1);
+    }
+
+    @Override
+    protected void writeByteArray(byte[] value, int offset, int length) throws IOException {
+        ensureCapacity(length);
+        int position = nioBuffer.position();
+        UnsafeDirectBufferUtil.setBytes(address(position), value, offset, length);
+        nioBuffer.position(position + length);
+    }
+
+    @Override
+    protected void ensureCapacity(int required) throws ProtocolException {
+        if (nioBuffer.remaining() < required) {
+            int position = nioBuffer.position();
+
+            while (capacity - position < required) {
+                if (capacity == maxCapacity) {
+                    throw new ProtocolException("Buffer overflow. Available: " + (capacity - position) + ", required: "
+                                                + required);
+                }
+                capacity = Math.min(capacity << 1, maxCapacity);
+                if (capacity < 0) {
+                    capacity = maxCapacity;
+                }
+            }
+
+            nioBuffer = outputBuf.nioByteBuffer(capacity - position);
+            capacity = nioBuffer.limit();
+            // Need to update the direct buffer's memory address
+            updateBufferAddress();
+        }
+    }
+
+    private void updateBufferAddress() {
+        memoryAddress = UnsafeUtil.addressOffset(nioBuffer);
+    }
+
+    private long address(int position) {
+        return memoryAddress + position;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/InputBuf.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/InputBuf.java
new file mode 100644
index 0000000..d4149e8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/InputBuf.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.io;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface InputBuf {
+
+    /**
+     * Exposes this backing data's readable bytes as an {@link InputStream}.
+     */
+    InputStream inputStream();
+
+    /**
+     * Exposes this backing data's readable bytes as a NIO {@link ByteBuffer}.
+     */
+    ByteBuffer nioByteBuffer();
+
+    /**
+     * Returns the number of readable bytes.
+     */
+    int size();
+
+    /**
+     * Returns {@code true} if and only if this buf has a reference to the low-level memory address that points
+     * to the backing data.
+     */
+    boolean hasMemoryAddress();
+
+    /**
+     * Decreases the reference count by {@code 1} and deallocates this object if the reference count reaches at
+     * {@code 0}.
+     */
+    boolean release();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputBuf.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputBuf.java
new file mode 100644
index 0000000..b7075f1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputBuf.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.io;
+
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface OutputBuf {
+
+    /**
+     * Exposes this backing data as an {@link OutputStream}.
+     */
+    OutputStream outputStream();
+
+    /**
+     * Exposes this backing data as a NIO {@link ByteBuffer}.
+     */
+    ByteBuffer nioByteBuffer(final int minWritableBytes);
+
+    /**
+     * Returns the number of readable bytes.
+     */
+    int size();
+
+    /**
+     * Returns {@code true} if and only if this buf has a reference to the low-level memory address that points
+     * to the backing data.
+     */
+    boolean hasMemoryAddress();
+
+    /**
+     * Returns the backing object.
+     */
+    Object backingObject();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputStreams.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputStreams.java
new file mode 100644
index 0000000..a296431
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/io/OutputStreams.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization.io;
+
+import java.io.ByteArrayOutputStream;
+
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class OutputStreams {
+
+    private static final ReferenceFieldUpdater bufUpdater     = Updaters
+                                                                                                 .newReferenceFieldUpdater(
+                                                                                                     ByteArrayOutputStream.class,
+                                                                                                     "buf");
+
+    // Reuse the byte[] in ByteArrayOutputStream
+    private static final ThreadLocal                   bufThreadLocal = ThreadLocal.withInitial(
+                                                                                                 () -> new ByteArrayOutputStream(Serializer.DEFAULT_BUF_SIZE));
+
+    public static ByteArrayOutputStream getByteArrayOutputStream() {
+        return bufThreadLocal.get();
+    }
+
+    public static void resetBuf(final ByteArrayOutputStream buf) {
+        buf.reset(); // for reuse
+
+        // prevent large blocks of memory from being held strong reference
+        if (bufUpdater.get(buf).length > Serializer.MAX_CACHED_BUF_SIZE) {
+            bufUpdater.set(buf, new byte[Serializer.DEFAULT_BUF_SIZE]);
+        }
+    }
+
+    private OutputStreams() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/package-info.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/package-info.java
new file mode 100644
index 0000000..c4bba2a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/serialization/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Introducing this package to prepare for serialization/deserialization
+ * of zero copies.
+ *
+ * @author jiachun.fjc
+ */
+package com.alipay.sofa.jraft.rhea.serialization;
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/AbstractKVStoreSnapshotFile.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/AbstractKVStoreSnapshotFile.java
new file mode 100644
index 0000000..3ba6599
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/AbstractKVStoreSnapshotFile.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.zip.Checksum;
+
+import com.alipay.sofa.jraft.rhea.storage.zip.ZipStrategyManager;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.CRC64;
+import com.alipay.sofa.jraft.util.Requires;
+import com.google.protobuf.ByteString;
+
+import static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+
+/**
+ * @author jiachun.fjc
+ */
+public abstract class AbstractKVStoreSnapshotFile implements KVStoreSnapshotFile {
+
+    private static final Logger LOG              = LoggerFactory.getLogger(AbstractKVStoreSnapshotFile.class);
+
+    private static final String SNAPSHOT_DIR     = "kv";
+    private static final String SNAPSHOT_ARCHIVE = "kv.zip";
+
+    protected final Serializer  serializer       = Serializers.getDefault();
+
+    @Override
+    public void save(final SnapshotWriter writer, final Region region, final Closure done,
+                     final ExecutorService executor) {
+        final String writerPath = writer.getPath();
+        final String snapshotPath = Paths.get(writerPath, SNAPSHOT_DIR).toString();
+        try {
+            doSnapshotSave(snapshotPath, region, executor).whenComplete((metaBuilder, throwable) -> {
+                if (throwable == null) {
+                    executor.execute(() -> compressSnapshot(writer, metaBuilder, done));
+                } else {
+                    LOG.error("Fail to save snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
+                            StackTraceUtil.stackTrace(throwable));
+                    done.run(new Status(RaftError.EIO, "Fail to save snapshot at %s, error is %s", writerPath,
+                            throwable.getMessage()));
+                }
+            });
+        } catch (final Throwable t) {
+            LOG.error("Fail to save snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
+                    StackTraceUtil.stackTrace(t));
+            done.run(new Status(RaftError.EIO, "Fail to save snapshot at %s, error is %s", writerPath,
+                    t.getMessage()));
+        }
+    }
+
+    @Override
+    public boolean load(final SnapshotReader reader, final Region region) {
+        final LocalFileMeta meta = (LocalFileMeta) reader.getFileMeta(SNAPSHOT_ARCHIVE);
+        final String readerPath = reader.getPath();
+        if (meta == null) {
+            LOG.error("Can't find kv snapshot file, path={}.", readerPath);
+            return false;
+        }
+        final String snapshotPath = Paths.get(readerPath, SNAPSHOT_DIR).toString();
+        try {
+            decompressSnapshot(readerPath, meta);
+            doSnapshotLoad(snapshotPath, meta, region);
+            final File tmp = new File(snapshotPath);
+            // Delete the decompressed temporary file. If the deletion fails (although it is a small probability
+            // event), it may affect the next snapshot decompression result. Therefore, the safest way is to
+            // terminate the state machine immediately. Users can choose to manually delete and restart according
+            // to the log information.
+            if (tmp.exists()) {
+                FileUtils.forceDelete(new File(snapshotPath));
+            }
+            return true;
+        } catch (final Throwable t) {
+            LOG.error("Fail to load snapshot, path={}, file list={}, {}.", readerPath, reader.listFiles(),
+                StackTraceUtil.stackTrace(t));
+            return false;
+        }
+    }
+
+    abstract CompletableFuture doSnapshotSave(final String snapshotPath, final Region region,
+                                                                     final ExecutorService executor) throws Exception;
+
+    abstract void doSnapshotLoad(final String snapshotPath, final LocalFileMeta meta, final Region region)
+                                                                                                          throws Exception;
+
+    protected void compressSnapshot(final SnapshotWriter writer, final LocalFileMeta.Builder metaBuilder,
+                                    final Closure done) {
+        final String writerPath = writer.getPath();
+        final String outputFile = Paths.get(writerPath, SNAPSHOT_ARCHIVE).toString();
+        try {
+            final Checksum checksum = new CRC64();
+            ZipStrategyManager.getDefault().compress(writerPath, SNAPSHOT_DIR, outputFile, checksum);
+            metaBuilder.setChecksum(Long.toHexString(checksum.getValue()));
+            if (writer.addFile(SNAPSHOT_ARCHIVE, metaBuilder.build())) {
+                done.run(Status.OK());
+            } else {
+                done.run(new Status(RaftError.EIO, "Fail to add snapshot file: %s", writerPath));
+            }
+        } catch (final Throwable t) {
+            LOG.error("Fail to compress snapshot, path={}, file list={}, {}.", writerPath, writer.listFiles(),
+                StackTraceUtil.stackTrace(t));
+            done.run(new Status(RaftError.EIO, "Fail to compress snapshot at %s, error is %s", writerPath, t
+                .getMessage()));
+        }
+    }
+
+    protected void decompressSnapshot(final String readerPath, final LocalFileMeta meta) throws Throwable {
+        final String sourceFile = Paths.get(readerPath, SNAPSHOT_ARCHIVE).toString();
+        final Checksum checksum = new CRC64();
+        ZipStrategyManager.getDefault().deCompress(sourceFile, readerPath, checksum);
+        if (meta.hasChecksum()) {
+            Requires.requireTrue(meta.getChecksum().equals(Long.toHexString(checksum.getValue())),
+                "Snapshot checksum failed");
+        }
+    }
+
+    protected  T readMetadata(final LocalFileMeta meta, final Class cls) {
+        final ByteString userMeta = meta.getUserMeta();
+        return this.serializer.readObject(userMeta.toByteArray(), cls);
+    }
+
+    protected  LocalFileMeta.Builder writeMetadata(final T metadata) {
+        if (metadata == null) {
+            return LocalFileMeta.newBuilder();
+        }
+        return LocalFileMeta.newBuilder() //
+            .setUserMeta(ByteString.copyFrom(this.serializer.writeObject(metadata)));
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseKVStoreClosure.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseKVStoreClosure.java
new file mode 100644
index 0000000..f28675f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseKVStoreClosure.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public abstract class BaseKVStoreClosure implements KVStoreClosure {
+
+    private volatile Errors error;
+    private volatile Object data;
+
+    @Override
+    public Errors getError() {
+        return error;
+    }
+
+    @Override
+    public void setError(Errors error) {
+        this.error = error;
+    }
+
+    @Override
+    public Object getData() {
+        return data;
+    }
+
+    @Override
+    public void setData(Object data) {
+        this.data = data;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseRawKVStore.java
new file mode 100644
index 0000000..8a9eee1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BaseRawKVStore.java
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.StorageException;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.codahale.metrics.Timer;
+
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.DB_TIMER;
+
+/**
+ * @author jiachun.fjc
+ */
+public abstract class BaseRawKVStore implements RawKVStore, Lifecycle {
+
+    @Override
+    public void get(final byte[] key, final KVStoreClosure closure) {
+        get(key, true, closure);
+    }
+
+    @Override
+    public void multiGet(final List keys, final KVStoreClosure closure) {
+        multiGet(keys, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final boolean returnValue, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void execute(final NodeExecutor nodeExecutor, final boolean isLeader, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("EXECUTE");
+        try {
+            if (nodeExecutor != null) {
+                nodeExecutor.execute(Status.OK(), isLeader);
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            final Logger LOG = LoggerFactory.getLogger(getClass());
+            LOG.error("Fail to [EXECUTE], {}.", StackTraceUtil.stackTrace(e));
+            if (nodeExecutor != null) {
+                nodeExecutor.execute(new Status(RaftError.EIO, "Fail to [EXECUTE]"), isLeader);
+            }
+            setCriticalError(closure, "Fail to [EXECUTE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    public long getSafeEndValueForSequence(final long startVal, final int step) {
+        return Math.max(startVal, Long.MAX_VALUE - step < startVal ? Long.MAX_VALUE : startVal + step);
+    }
+
+    /**
+     * If limit == 0, it will be modified to Integer.MAX_VALUE on the server
+     * and then queried.  So 'limit == 0' means that the number of queries is
+     * not limited. This is because serialization uses varint to compress
+     * numbers.  In the case of 0, only 1 byte is occupied, and Integer.MAX_VALUE
+     * takes 5 bytes.
+     *
+     * @param limit input limit
+     * @return normalize limit
+     */
+    protected int normalizeLimit(final int limit) {
+        return limit > 0 ? limit : Integer.MAX_VALUE;
+    }
+
+    /**
+     * Note: This is not a very precise behavior, don't rely on its accuracy.
+     */
+    public abstract long getApproximateKeysInRange(final byte[] startKey, final byte[] endKey);
+
+    /**
+     * Note: This is not a very precise behavior, don't rely on its accuracy.
+     */
+    public abstract byte[] jumpOver(final byte[] startKey, final long distance);
+
+    /**
+     * Init the fencing token of new region.
+     *
+     * @param parentKey the fencing key of parent region
+     * @param childKey  the fencing key of new region
+     */
+    public abstract void initFencingToken(final byte[] parentKey, final byte[] childKey);
+
+    // static methods
+    //
+    static Timer.Context getTimeContext(final String opName) {
+        return KVMetrics.timer(DB_TIMER, opName).time();
+    }
+
+    /**
+     * Sets success, if current node is a leader, reply to
+     * client success with result data response.
+     *
+     * @param closure callback
+     * @param data    result data to reply to client
+     */
+    static void setSuccess(final KVStoreClosure closure, final Object data) {
+        if (closure != null) {
+            // closure is null on follower node
+            closure.setData(data);
+            closure.run(Status.OK());
+        }
+    }
+
+    /**
+     * Sets failure, if current node is a leader, reply to
+     * client failure response.
+     *
+     * @param closure callback
+     * @param message error message
+     */
+    static void setFailure(final KVStoreClosure closure, final String message) {
+        if (closure != null) {
+            // closure is null on follower node
+            closure.setError(Errors.STORAGE_ERROR);
+            closure.run(new Status(RaftError.EIO, message));
+        }
+    }
+
+    /**
+     * Sets critical error and halt the state machine.
+     *
+     * If current node is a leader, first reply to client
+     * failure response.
+     *
+     * @param closure callback
+     * @param message error message
+     * @param error   critical error
+     */
+    static void setCriticalError(final KVStoreClosure closure, final String message, final Throwable error) {
+        // Will call closure#run in FSMCaller
+        setClosureError(closure);
+        if (error != null) {
+            throw new StorageException(message, error);
+        }
+    }
+
+    /**
+     * Sets critical error and halt the state machine.
+     *
+     * If current node is a leader, first reply to client
+     * failure response.
+     *
+     * @param closures callback list
+     * @param message  error message
+     * @param error    critical error
+     */
+    static void setCriticalError(final List closures, final String message, final Throwable error) {
+        for (final KVStoreClosure closure : closures) {
+            // Will call closure#run in FSMCaller
+            setClosureError(closure);
+        }
+        if (error != null) {
+            throw new StorageException(message, error);
+        }
+    }
+
+    static void setClosureError(final KVStoreClosure closure) {
+        if (closure != null) {
+            // closure is null on follower node
+            closure.setError(Errors.STORAGE_ERROR);
+        }
+    }
+
+    /**
+     * Sets the result first, then gets it by {@link #getData(KVStoreClosure)}
+     * when we ensure successful.
+     *
+     * @param closure callback
+     * @param data    data
+     */
+    static void setData(final KVStoreClosure closure, final Object data) {
+        if (closure != null) {
+            // closure is null on follower node
+            closure.setData(data);
+        }
+    }
+
+    /**
+     * Gets data from closure.
+     *
+     * @param closure callback
+     * @return closure data
+     */
+    static Object getData(final KVStoreClosure closure) {
+        return closure == null ? null : closure.getData();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BatchRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BatchRawKVStore.java
new file mode 100644
index 0000000..7830730
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/BatchRawKVStore.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * The default batch write implementation, without any optimization,
+ * subclasses need to override and optimize.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class BatchRawKVStore extends BaseRawKVStore {
+
+    public void batchPut(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            put(op.getKey(), op.getValue(), kvState.getDone());
+        }
+    }
+
+    public void batchPutIfAbsent(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            putIfAbsent(op.getKey(), op.getValue(), kvState.getDone());
+        }
+    }
+
+    public void batchPutList(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            put(kvState.getOp().getEntries(), kvState.getDone());
+        }
+    }
+
+    public void batchDelete(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            delete(kvState.getOp().getKey(), kvState.getDone());
+        }
+    }
+
+    public void batchDeleteRange(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            deleteRange(op.getStartKey(), op.getEndKey(), kvState.getDone());
+        }
+    }
+
+    public void batchDeleteList(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            delete(kvState.getOp().getKeys(), kvState.getDone());
+        }
+    }
+
+    public void batchGetSequence(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            getSequence(op.getSeqKey(), op.getStep(), kvState.getDone());
+        }
+    }
+
+    public void batchNodeExecute(final KVStateOutputList kvStates, final boolean isLeader) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            execute(kvState.getOp().getNodeExecutor(), isLeader, kvState.getDone());
+        }
+    }
+
+    public void batchTryLockWith(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            final Pair acquirerPair = op.getAcquirerPair();
+            tryLockWith(op.getKey(), op.getFencingKey(), acquirerPair.getKey(), acquirerPair.getValue(),
+                kvState.getDone());
+        }
+    }
+
+    public void batchReleaseLockWith(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            releaseLockWith(op.getKey(), op.getAcquirer(), kvState.getDone());
+        }
+    }
+
+    public void batchGet(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            get(kvState.getOp().getKey(), kvState.getDone());
+        }
+    }
+
+    public void batchMultiGet(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            multiGet(kvState.getOp().getKeyList(), kvState.getDone());
+        }
+    }
+
+    public void batchContainsKey(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            containsKey(kvState.getOp().getKey(), kvState.getDone());
+        }
+    }
+
+    public void batchScan(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            scan(op.getStartKey(), op.getEndKey(), op.getLimit(), true, op.isReturnValue(), kvState.getDone());
+        }
+    }
+
+    public void batchReverseScan(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            reverseScan(op.getStartKey(), op.getEndKey(), op.getLimit(), true, op.isReturnValue(), kvState.getDone());
+        }
+    }
+
+    public void batchGetAndPut(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            getAndPut(op.getKey(), op.getValue(), kvState.getDone());
+        }
+    }
+
+    public void batchCompareAndPut(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            compareAndPut(op.getKey(), op.getExpect(), op.getValue(), kvState.getDone());
+        }
+    }
+
+    public void batchCompareAndPutAll(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            compareAndPutAll(kvState.getOp().getCASEntries(), kvState.getDone());
+        }
+    }
+
+    public void batchMerge(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            final KVOperation op = kvState.getOp();
+            merge(op.getKey(), op.getValue(), kvState.getDone());
+        }
+    }
+
+    public void batchResetSequence(final KVStateOutputList kvStates) {
+        for (int i = 0, l = kvStates.size(); i < l; i++) {
+            final KVState kvState = kvStates.get(i);
+            resetSequence(kvState.getOp().getKey(), kvState.getDone());
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/CASEntry.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/CASEntry.java
new file mode 100644
index 0000000..bd5d41f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/CASEntry.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class CASEntry implements Serializable {
+
+    private static final long serialVersionUID = -8024887237976722615L;
+
+    private byte[]            key;
+    private byte[]            expect;
+    private byte[]            update;
+
+    public CASEntry() {
+    }
+
+    public CASEntry(byte[] key, byte[] expect, byte[] update) {
+        this.key = key;
+        this.expect = expect;
+        this.update = update;
+    }
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getExpect() {
+        return expect;
+    }
+
+    public void setExpect(byte[] expect) {
+        this.expect = expect;
+    }
+
+    public byte[] getUpdate() {
+        return update;
+    }
+
+    public void setUpdate(byte[] update) {
+        this.update = update;
+    }
+
+    public int length() {
+        return (this.key == null ? 0 : this.key.length) + (this.expect == null ? 0 : this.expect.length)
+               + (this.update == null ? 0 : this.update.length);
+    }
+
+    @Override
+    public String toString() {
+        return "CASEntry{" + "key=" + BytesUtil.toHex(key) + ", expect=" + BytesUtil.toHex(expect) + ", update="
+               + BytesUtil.toHex(update) + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVClosureAdapter.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVClosureAdapter.java
new file mode 100644
index 0000000..5be393f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVClosureAdapter.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class KVClosureAdapter implements KVStoreClosure {
+
+    private static final Logger LOG = LoggerFactory.getLogger(KVClosureAdapter.class);
+
+    private KVStoreClosure      done;
+    private KVOperation         operation;
+
+    public KVClosureAdapter(final KVStoreClosure done, final KVOperation operation) {
+        this.done = done;
+        this.operation = operation;
+    }
+
+    public KVStoreClosure getDone() {
+        return done;
+    }
+
+    public KVOperation getOperation() {
+        return operation;
+    }
+
+    @Override
+    public void run(final Status status) {
+        if (status.isOk()) {
+            setError(Errors.NONE);
+        } else {
+            LOG.error("Fail status: {}.", status);
+            if (getError() == null) {
+                switch (status.getRaftError()) {
+                    case SUCCESS:
+                        setError(Errors.NONE);
+                        break;
+                    case EINVAL:
+                        setError(Errors.INVALID_REQUEST);
+                        break;
+                    case EIO:
+                        setError(Errors.STORAGE_ERROR);
+                        break;
+                    default:
+                        setError(Errors.LEADER_NOT_AVAILABLE);
+                        break;
+                }
+            }
+        }
+        if (done != null) {
+            done.run(status);
+        }
+        reset();
+    }
+
+    @Override
+    public Errors getError() {
+        if (this.done != null) {
+            return this.done.getError();
+        }
+        return null;
+    }
+
+    @Override
+    public void setError(Errors error) {
+        if (this.done != null) {
+            this.done.setError(error);
+        }
+    }
+
+    @Override
+    public Object getData() {
+        if (this.done != null) {
+            return this.done.getData();
+        }
+        return null;
+    }
+
+    @Override
+    public void setData(Object data) {
+        if (this.done != null) {
+            this.done.setData(data);
+        }
+    }
+
+    private void reset() {
+        done = null;
+        operation = null;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVEntry.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVEntry.java
new file mode 100644
index 0000000..e8107ff
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVEntry.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class KVEntry implements Serializable {
+
+    private static final long serialVersionUID = -5678680976506834026L;
+
+    private byte[]            key;
+    private byte[]            value;
+
+    public KVEntry() {
+    }
+
+    public KVEntry(byte[] key, byte[] value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public void setKey(byte[] key) {
+        this.key = key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public void setValue(byte[] value) {
+        this.value = value;
+    }
+
+    public int length() {
+        return (this.key == null ? 0 : this.key.length) + (this.value == null ? 0 : this.value.length);
+    }
+
+    @Override
+    public String toString() {
+        return "KVEntry{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVIterator.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVIterator.java
new file mode 100644
index 0000000..58d5f93
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVIterator.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ * A heap-allocated iterator over the contents of the database.
+ *
+ * Caller should close the iterator when it is no longer needed.
+ * The returned iterator should be closed before this db is closed.
+ *
+ * 
+ *     KVIterator it = unsafeLocalIterator();
+ *     try {
+ *         // do something
+ *     } finally {
+ *         it.close();
+ *     }
+ * 
+ *
+ * @author jiachun.fjc
+ */
+public interface KVIterator extends AutoCloseable {
+
+    /**
+     * An iterator is either positioned at an entry, or not valid.
+     * This method returns true if the iterator is valid.
+     *
+     * @return true if iterator is valid.
+     */
+    boolean isValid();
+
+    /**
+     * Position at the first entry in the source.  The iterator is Valid()
+     * after this call if the source is not empty.
+     */
+    void seekToFirst();
+
+    /**
+     * Position at the last entry in the source.  The iterator is
+     * valid after this call if the source is not empty.
+     */
+    void seekToLast();
+
+    /**
+     * Position at the first entry in the source whose key is that or
+     * past target.
+     *
+     * The iterator is valid after this call if the source contains
+     * a key that comes at or past target.
+     *
+     * @param target byte array describing a key or a
+     *               key prefix to seek for.
+     */
+    void seek(final byte[] target);
+
+    /**
+     * Position at the first entry in the source whose key is that or
+     * before target.
+     *
+     * The iterator is valid after this call if the source contains
+     * a key that comes at or before target.
+     *
+     * @param target byte array describing a key or a
+     *               key prefix to seek for.
+     */
+    void seekForPrev(final byte[] target);
+
+    /**
+     * Moves to the next entry in the source.  After this call, Valid() is
+     * true if the iterator was not positioned at the last entry in the source.
+     *
+     * REQUIRES: {@link #isValid()}
+     */
+    void next();
+
+    /**
+     * Moves to the previous entry in the source.  After this call, Valid() is
+     * true if the iterator was not positioned at the first entry in source.
+     *
+     * REQUIRES: {@link #isValid()}
+     */
+    void prev();
+
+    /**
+     * Return the key for the current entry.  The underlying storage for
+     * the returned slice is valid only until the next modification of
+     * the iterator.
+     *
+     * REQUIRES: {@link #isValid()}
+     *
+     * @return key for the current entry.
+     */
+    byte[] key();
+
+    /**
+     * Return the value for the current entry.  The underlying storage for
+     * the returned slice is valid only until the next modification of
+     * the iterator.
+     *
+     * REQUIRES: !AtEnd() && !AtStart()
+     *
+     * @return value for the current entry.
+     */
+    byte[] value();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVOperation.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVOperation.java
new file mode 100644
index 0000000..571827b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVOperation.java
@@ -0,0 +1,423 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * The KV store operation
+ *
+ * @author jiachun.fjc
+ */
+public class KVOperation implements Serializable {
+
+    private static final long   serialVersionUID = 1368415383186519279L;
+
+    /** Encode magic number */
+    public static final byte    MAGIC            = 0x00;
+
+    /** Put operation */
+    public static final byte    PUT              = 0x01;
+    /** PutIfAbsent operation */
+    public static final byte    PUT_IF_ABSENT    = 0x02;
+    /** Delete operation */
+    public static final byte    DELETE           = 0x03;
+    /** Put list operation */
+    public static final byte    PUT_LIST         = 0x04;
+    /** Delete range operation */
+    public static final byte    DELETE_RANGE     = 0x05;
+    /** Get sequence operation */
+    public static final byte    GET_SEQUENCE     = 0x06;
+    /** Execute on every node operation */
+    public static final byte    NODE_EXECUTE     = 0x07;
+    /** Tries to lock the specified key */
+    public static final byte    KEY_LOCK         = 0x08;
+    /** Unlock the specified key */
+    public static final byte    KEY_LOCK_RELEASE = 0x09;
+    /** Get operation */
+    public static final byte    GET              = 0x0a;
+    /** MultiGet operation  */
+    public static final byte    MULTI_GET        = 0x0b;
+    /** Scan operation */
+    public static final byte    SCAN             = 0x0c;
+    /** Get and put operation */
+    public static final byte    GET_PUT          = 0x0d;
+    /** Merge operation */
+    public static final byte    MERGE            = 0x0e;
+    /** Reset sequence operation */
+    public static final byte    RESET_SEQUENCE   = 0x0f;
+
+    // split operation ***********************************
+    /** Range split operation */
+    public static final byte    RANGE_SPLIT      = 0x10;
+    /** Compare and put operation */
+    public static final byte    COMPARE_PUT      = 0x11;
+    /** Delete list operation */
+    public static final byte    DELETE_LIST      = 0x12;
+    /** Contains key operation */
+    public static final byte    CONTAINS_KEY     = 0x13;
+
+    /** Reverse Scan operation */
+    public static final byte    REVERSE_SCAN     = 0x14;
+    /** Compare and put all */
+    public static final byte    COMPARE_PUT_ALL  = 0x15;
+
+    public static final byte    EOF              = 0x16;
+
+    private static final byte[] VALID_OPS;
+
+    static {
+        VALID_OPS = new byte[EOF - 1];
+        VALID_OPS[0] = PUT;
+        VALID_OPS[1] = PUT_IF_ABSENT;
+        VALID_OPS[2] = DELETE;
+        VALID_OPS[3] = PUT_LIST;
+        VALID_OPS[4] = DELETE_RANGE;
+        VALID_OPS[5] = GET_SEQUENCE;
+        VALID_OPS[6] = NODE_EXECUTE;
+        VALID_OPS[7] = KEY_LOCK;
+        VALID_OPS[8] = KEY_LOCK_RELEASE;
+        VALID_OPS[9] = GET;
+        VALID_OPS[10] = MULTI_GET;
+        VALID_OPS[11] = SCAN;
+        VALID_OPS[12] = GET_PUT;
+        VALID_OPS[13] = MERGE;
+        VALID_OPS[14] = RESET_SEQUENCE;
+        VALID_OPS[15] = RANGE_SPLIT;
+        VALID_OPS[16] = COMPARE_PUT;
+        VALID_OPS[17] = DELETE_LIST;
+        VALID_OPS[18] = CONTAINS_KEY;
+        VALID_OPS[19] = REVERSE_SCAN;
+        VALID_OPS[20] = COMPARE_PUT_ALL;
+    }
+
+    private byte[]              key;                                    // also startKey for DELETE_RANGE
+    private byte[]              value;                                  // also endKey for DELETE_RANGE
+    private Object              attach;
+
+    private byte                op;
+
+    public static boolean isValidOp(final byte op) {
+        return op > MAGIC && op < EOF;
+    }
+
+    /**
+     * The best practice is to call this method only once
+     * and keep a copy of it yourself.
+     */
+    public static byte[] getValidOps() {
+        final byte[] copy = new byte[VALID_OPS.length];
+        System.arraycopy(VALID_OPS, 0, copy, 0, copy.length);
+        return copy;
+    }
+
+    public static KVOperation createPut(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, null, PUT);
+    }
+
+    public static KVOperation createPutIfAbsent(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, null, PUT_IF_ABSENT);
+    }
+
+    public static KVOperation createDelete(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, BytesUtil.EMPTY_BYTES, null, DELETE);
+    }
+
+    public static KVOperation createPutList(final List entries) {
+        Requires.requireNonNull(entries, "entries");
+        Requires.requireTrue(!entries.isEmpty(), "entries is empty");
+        return new KVOperation(BytesUtil.EMPTY_BYTES, BytesUtil.EMPTY_BYTES, entries, PUT_LIST);
+    }
+
+    public static KVOperation createDeleteRange(final byte[] startKey, final byte[] endKey) {
+        Requires.requireNonNull(startKey, "startKey");
+        Requires.requireNonNull(endKey, "endKey");
+        return new KVOperation(startKey, endKey, null, DELETE_RANGE);
+    }
+
+    public static KVOperation createDeleteList(final List keys) {
+        Requires.requireNonNull(keys, "keys");
+        Requires.requireTrue(!keys.isEmpty(), "keys is empty");
+        return new KVOperation(BytesUtil.EMPTY_BYTES, BytesUtil.EMPTY_BYTES, keys, DELETE_LIST);
+    }
+
+    public static KVOperation createGetSequence(final byte[] seqKey, final int step) {
+        Requires.requireNonNull(seqKey, "seqKey");
+        Requires.requireTrue(step > 0, "step must > 0");
+        return new KVOperation(seqKey, BytesUtil.EMPTY_BYTES, step, GET_SEQUENCE);
+    }
+
+    public static KVOperation createNodeExecutor(final NodeExecutor nodeExecutor) {
+        return new KVOperation(BytesUtil.EMPTY_BYTES, BytesUtil.EMPTY_BYTES, nodeExecutor, NODE_EXECUTE);
+    }
+
+    public static KVOperation createKeyLockRequest(final byte[] key, final byte[] fencingKey,
+                                                   final Pair acquirerPair) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, fencingKey, acquirerPair, KEY_LOCK);
+    }
+
+    public static KVOperation createKeyLockReleaseRequest(final byte[] key, final DistributedLock.Acquirer acquirer) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, BytesUtil.EMPTY_BYTES, acquirer, KEY_LOCK_RELEASE);
+    }
+
+    public static KVOperation createGet(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, BytesUtil.EMPTY_BYTES, null, GET);
+    }
+
+    public static KVOperation createMultiGet(final List keys) {
+        Requires.requireNonNull(keys, "keys");
+        Requires.requireTrue(!keys.isEmpty(), "keys is empty");
+        return new KVOperation(BytesUtil.EMPTY_BYTES, BytesUtil.EMPTY_BYTES, keys, MULTI_GET);
+    }
+
+    public static KVOperation createContainsKey(final byte[] key) {
+        Requires.requireNonNull(key, "key");
+        return new KVOperation(key, BytesUtil.EMPTY_BYTES, null, CONTAINS_KEY);
+    }
+
+    public static KVOperation createScan(final byte[] startKey, final byte[] endKey, final int limit,
+                                         final boolean returnValue) {
+        return new KVOperation(startKey, endKey, Pair.of(limit, returnValue), SCAN);
+    }
+
+    public static KVOperation createReverseScan(final byte[] startKey, final byte[] endKey, final int limit,
+                                                final boolean returnValue) {
+        return new KVOperation(startKey, endKey, Pair.of(limit, returnValue), REVERSE_SCAN);
+    }
+
+    public static KVOperation createGetAndPut(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, null, GET_PUT);
+    }
+
+    public static KVOperation createCompareAndPut(final byte[] key, final byte[] expect, final byte[] update) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(expect, "expect");
+        Requires.requireNonNull(update, "update");
+        return new KVOperation(key, update, expect, COMPARE_PUT);
+    }
+
+    public static KVOperation createMerge(final byte[] key, final byte[] value) {
+        Requires.requireNonNull(key, "key");
+        Requires.requireNonNull(value, "value");
+        return new KVOperation(key, value, null, MERGE);
+    }
+
+    public static KVOperation createResetSequence(final byte[] seqKey) {
+        Requires.requireNonNull(seqKey, "seqKey");
+        return new KVOperation(seqKey, BytesUtil.EMPTY_BYTES, null, RESET_SEQUENCE);
+    }
+
+    public static KVOperation createRangeSplit(final byte[] splitKey, final long currentRegionId, final long newRegionId) {
+        Requires.requireNonNull(splitKey, "splitKey");
+        return new KVOperation(splitKey, BytesUtil.EMPTY_BYTES, Pair.of(currentRegionId, newRegionId), RANGE_SPLIT);
+    }
+
+    public static KVOperation createCompareAndPutAll(final List entries) {
+        Requires.requireNonNull(entries, "entries");
+        Requires.requireTrue(!entries.isEmpty(), "entries is empty");
+        return new KVOperation(BytesUtil.EMPTY_BYTES, BytesUtil.EMPTY_BYTES, entries, COMPARE_PUT_ALL);
+    }
+
+    public KVOperation() {
+    }
+
+    public KVOperation(byte[] key, byte[] value, Object attach, byte op) {
+        this.key = key;
+        this.value = value;
+        this.attach = attach;
+        this.op = op;
+    }
+
+    public byte[] getKey() {
+        return key;
+    }
+
+    public byte[] getStartKey() {
+        return key;
+    }
+
+    public byte[] getSeqKey() {
+        return key;
+    }
+
+    public byte[] getValue() {
+        return value;
+    }
+
+    public byte[] getEndKey() {
+        return value;
+    }
+
+    public byte[] getFencingKey() {
+        return value;
+    }
+
+    public byte getOp() {
+        return op;
+    }
+
+    public boolean isReadOp() {
+        return GET == this.op || MULTI_GET == this.op || SCAN == this.op || CONTAINS_KEY == this.op
+               || REVERSE_SCAN == this.op;
+    }
+
+    public int getStep() {
+        return (Integer) this.attach;
+    }
+
+    public byte[] getExpect() {
+        return (byte[]) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public List getEntries() {
+        return (List) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public List getCASEntries() {
+        return (List) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public List getKeys() {
+        return (List) this.attach;
+    }
+
+    public NodeExecutor getNodeExecutor() {
+        return (NodeExecutor) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public Pair getAcquirerPair() {
+        return (Pair) this.attach;
+    }
+
+    public DistributedLock.Acquirer getAcquirer() {
+        return (DistributedLock.Acquirer) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public List getKeyList() {
+        return (List) this.attach;
+    }
+
+    @SuppressWarnings("unchecked")
+    public long getCurrentRegionId() {
+        return ((Pair) this.attach).getKey();
+    }
+
+    @SuppressWarnings("unchecked")
+    public long getNewRegionId() {
+        return ((Pair) this.attach).getValue();
+    }
+
+    @SuppressWarnings("unchecked")
+    public int getLimit() {
+        if (this.attach instanceof Pair) {
+            return ((Pair) this.attach).getKey();
+        } else {
+            // forwards compatibility
+            return (Integer) this.attach;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public boolean isReturnValue() {
+        if (this.attach instanceof Pair) {
+            return ((Pair) this.attach).getValue();
+        } else {
+            // forwards compatibility
+            return true;
+        }
+    }
+
+    public static String opName(KVOperation op) {
+        return opName(op.op);
+    }
+
+    public static String opName(byte op) {
+        switch (op) {
+            case PUT:
+                return "PUT";
+            case PUT_IF_ABSENT:
+                return "PUT_IF_ABSENT";
+            case DELETE:
+                return "DELETE";
+            case PUT_LIST:
+                return "PUT_LIST";
+            case DELETE_RANGE:
+                return "DELETE_RANGE";
+            case GET_SEQUENCE:
+                return "GET_SEQUENCE";
+            case NODE_EXECUTE:
+                return "NODE_EXECUTE";
+            case KEY_LOCK:
+                return "KEY_LOCK";
+            case KEY_LOCK_RELEASE:
+                return "KEY_LOCK_RELEASE";
+            case GET:
+                return "GET";
+            case MULTI_GET:
+                return "MULTI_GET";
+            case SCAN:
+                return "SCAN";
+            case GET_PUT:
+                return "GET_PUT";
+            case COMPARE_PUT:
+                return "COMPARE_PUT";
+            case MERGE:
+                return "MERGE";
+            case RESET_SEQUENCE:
+                return "RESET_SEQUENCE";
+            case RANGE_SPLIT:
+                return "RANGE_SPLIT";
+            case DELETE_LIST:
+                return "DELETE_LIST";
+            case CONTAINS_KEY:
+                return "CONTAINS_KEY";
+            case REVERSE_SCAN:
+                return "REVERSE_SCAN";
+            case COMPARE_PUT_ALL:
+                return "COMPARE_PUT_ALL";
+            default:
+                return "UNKNOWN" + op;
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "KVOperation{" + "key=" + BytesUtil.toHex(key) + ", value=" + BytesUtil.toHex(value) + ", attach="
+               + attach + ", op=" + op + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVState.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVState.java
new file mode 100644
index 0000000..99a2c28
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVState.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ * @author jiachun.fjc
+ */
+public class KVState {
+
+    private final KVOperation    op;
+    private final KVStoreClosure done;
+
+    public static KVState of(final KVOperation op, final KVStoreClosure done) {
+        return new KVState(op, done);
+    }
+
+    public KVState(KVOperation op, KVStoreClosure done) {
+        this.op = op;
+        this.done = done;
+    }
+
+    public boolean isSameOp(final KVOperation o) {
+        return this.op.getOp() == o.getOp();
+    }
+
+    public KVOperation getOp() {
+        return op;
+    }
+
+    public byte getOpByte() {
+        return this.op.getOp();
+    }
+
+    public KVStoreClosure getDone() {
+        return done;
+    }
+
+    @Override
+    public String toString() {
+        return "KVState{" + "op=" + op + ", done=" + done + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStateOutputList.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStateOutputList.java
new file mode 100644
index 0000000..240e924
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStateOutputList.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.RandomAccess;
+
+import com.alipay.sofa.jraft.util.Recyclable;
+import com.alipay.sofa.jraft.util.Recyclers;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * A simple kv state list which is recyclable.
+ * This implementation does not allow {@code null} elements to be added.
+ */
+public final class KVStateOutputList extends ArrayList implements Recyclable {
+
+    private static final long serialVersionUID         = -8605125654176467947L;
+
+    private static final int  DEFAULT_INITIAL_CAPACITY = 8;
+
+    /**
+     * Create a new empty {@link KVStateOutputList} instance
+     */
+    public static KVStateOutputList newInstance() {
+        return newInstance(DEFAULT_INITIAL_CAPACITY);
+    }
+
+    /**
+     * Create a new empty {@link KVStateOutputList} instance with the given capacity.
+     */
+    public static KVStateOutputList newInstance(final int minCapacity) {
+        final KVStateOutputList ret = recyclers.get();
+        ret.ensureCapacity(minCapacity);
+        return ret;
+    }
+
+    public boolean isSingletonList() {
+        return size() == 1;
+    }
+
+    /**
+     * You must first check to make sure that {@link #isSingletonList()}
+     * returns true.
+     */
+    public KVState getSingletonElement() {
+        Requires.requireTrue(!isEmpty(), "empty");
+        return get(0);
+    }
+
+    public KVState getFirstElement() {
+        return isEmpty() ? null : get(0);
+    }
+
+    @Override
+    public boolean addAll(final Collection c) {
+        checkNullElements(c);
+        return super.addAll(c);
+    }
+
+    @Override
+    public boolean addAll(final int index, final Collection c) {
+        checkNullElements(c);
+        return super.addAll(index, c);
+    }
+
+    private static void checkNullElements(final Collection c) {
+        if (c instanceof RandomAccess && c instanceof List) {
+            // produce less garbage
+            final List list = (List) c;
+            final int size = list.size();
+            for (int i = 0; i < size; i++) {
+                if (list.get(i) == null) {
+                    throw new IllegalArgumentException("c contains null values");
+                }
+            }
+        } else {
+            for (final Object element : c) {
+                if (element == null) {
+                    throw new IllegalArgumentException("c contains null values");
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean add(final KVState element) {
+        if (element == null) {
+            throw new NullPointerException("element");
+        }
+        return super.add(element);
+    }
+
+    @Override
+    public void add(final int index, final KVState element) {
+        if (element == null) {
+            throw new NullPointerException("element");
+        }
+        super.add(index, element);
+    }
+
+    @Override
+    public KVState set(final int index, final KVState element) {
+        if (element == null) {
+            throw new NullPointerException("element");
+        }
+        return super.set(index, element);
+    }
+
+    @Override
+    public boolean recycle() {
+        clear();
+        return recyclers.recycle(this, handle);
+    }
+
+    public static int threadLocalCapacity() {
+        return recyclers.threadLocalCapacity();
+    }
+
+    public static int threadLocalSize() {
+        return recyclers.threadLocalSize();
+    }
+
+    private KVStateOutputList(final Recyclers.Handle handle) {
+        this(handle, DEFAULT_INITIAL_CAPACITY);
+    }
+
+    private KVStateOutputList(final Recyclers.Handle handle, final int initialCapacity) {
+        super(initialCapacity);
+        this.handle = handle;
+    }
+
+    private transient final Recyclers.Handle          handle;
+
+    private static final Recyclers recyclers = new Recyclers(512) {
+
+                                                                    @Override
+                                                                    protected KVStateOutputList newObject(final Handle handle) {
+                                                                        return new KVStateOutputList(handle);
+                                                                    }
+                                                                };
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreClosure.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreClosure.java
new file mode 100644
index 0000000..a32416d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreClosure.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface KVStoreClosure extends Closure {
+
+    Errors getError();
+
+    void setError(final Errors error);
+
+    Object getData();
+
+    void setData(final Object data);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFile.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFile.java
new file mode 100644
index 0000000..b1974f1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFile.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.concurrent.ExecutorService;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface KVStoreSnapshotFile {
+
+    /**
+     * Save a snapshot for the specified region.
+     *
+     * @param writer   snapshot writer
+     * @param region   the region to save snapshot
+     * @param done     callback
+     * @param executor the executor to compress snapshot
+     */
+    void save(final SnapshotWriter writer, final Region region, final Closure done, final ExecutorService executor);
+
+    /**
+     * Load snapshot for the specified region.
+     *
+     * @param reader snapshot reader
+     * @param region the region to load snapshot
+     * @return true if load succeed
+     */
+    boolean load(final SnapshotReader reader, final Region region);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFileFactory.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFileFactory.java
new file mode 100644
index 0000000..e54b808
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreSnapshotFileFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class KVStoreSnapshotFileFactory {
+
+    public static  KVStoreSnapshotFile getKVStoreSnapshotFile(final BaseRawKVStore kvStore) {
+        Requires.requireNonNull(kvStore, "kvStore");
+        if (kvStore instanceof RocksRawKVStore) {
+            return new RocksKVStoreSnapshotFile((RocksRawKVStore) kvStore);
+        }
+        if (kvStore instanceof MemoryRawKVStore) {
+            return new MemoryKVStoreSnapshotFile((MemoryRawKVStore) kvStore);
+        }
+        throw reject("fail to find a KVStoreSnapshotFile with " + kvStore.getClass().getName());
+    }
+
+    private static UnsupportedOperationException reject(final String message) {
+        return new UnsupportedOperationException(message);
+    }
+
+    private KVStoreSnapshotFileFactory() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreStateMachine.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreStateMachine.java
new file mode 100644
index 0000000..ed18837
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/KVStoreStateMachine.java
@@ -0,0 +1,369 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+import com.alipay.sofa.jraft.entity.LeaderChangeContext;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rhea.StateListener;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.IllegalKVOperationException;
+import com.alipay.sofa.jraft.rhea.errors.StoreCodecException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.RecycleUtil;
+import com.codahale.metrics.Histogram;
+import com.codahale.metrics.Meter;
+
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.STATE_MACHINE_APPLY_QPS;
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.STATE_MACHINE_BATCH_WRITE;
+
+/**
+ * Rhea KV store state machine
+ *
+ * @author jiachun.fjc
+ */
+public class KVStoreStateMachine extends StateMachineAdapter {
+
+    private static final Logger       LOG        = LoggerFactory.getLogger(KVStoreStateMachine.class);
+
+    private final AtomicLong          leaderTerm = new AtomicLong(-1L);
+    private final Serializer          serializer = Serializers.getDefault();
+    private final Region              region;
+    private final StoreEngine         storeEngine;
+    private final BatchRawKVStore  rawKVStore;
+    private final KVStoreSnapshotFile storeSnapshotFile;
+    private final Meter               applyMeter;
+    private final Histogram           batchWriteHistogram;
+
+    public KVStoreStateMachine(Region region, StoreEngine storeEngine) {
+        this.region = region;
+        this.storeEngine = storeEngine;
+        this.rawKVStore = storeEngine.getRawKVStore();
+        this.storeSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.rawKVStore);
+        final String regionStr = String.valueOf(this.region.getId());
+        this.applyMeter = KVMetrics.meter(STATE_MACHINE_APPLY_QPS, regionStr);
+        this.batchWriteHistogram = KVMetrics.histogram(STATE_MACHINE_BATCH_WRITE, regionStr);
+    }
+
+    @Override
+    public void onApply(final Iterator it) {
+        int index = 0;
+        int applied = 0;
+        try {
+            KVStateOutputList kvStates = KVStateOutputList.newInstance();
+            while (it.hasNext()) {
+                KVOperation kvOp;
+                final KVClosureAdapter done = (KVClosureAdapter) it.done();
+                if (done != null) {
+                    kvOp = done.getOperation();
+                } else {
+                    final ByteBuffer buf = it.getData();
+                    try {
+                        if (buf.hasArray()) {
+                            kvOp = this.serializer.readObject(buf.array(), KVOperation.class);
+                        } else {
+                            kvOp = this.serializer.readObject(buf, KVOperation.class);
+                        }
+                        // follower ignore read operation
+                        if (kvOp != null && kvOp.isReadOp()) {
+                            ++index;
+                            it.next();
+                            continue;
+                        }
+                    } catch (final Throwable t) {
+                        ++index;
+                        throw new StoreCodecException("Decode operation error", t);
+                    }
+                }
+                final KVState first = kvStates.getFirstElement();
+                if (first != null && !first.isSameOp(kvOp)) {
+                    applied += batchApplyAndRecycle(first.getOpByte(), kvStates);
+                    kvStates = KVStateOutputList.newInstance();
+                }
+                kvStates.add(KVState.of(kvOp, done));
+                ++index;
+                it.next();
+            }
+            if (!kvStates.isEmpty()) {
+                final KVState first = kvStates.getFirstElement();
+                assert first != null;
+                applied += batchApplyAndRecycle(first.getOpByte(), kvStates);
+            }
+        } catch (final Throwable t) {
+            LOG.error("StateMachine meet critical error: {}.", StackTraceUtil.stackTrace(t));
+            it.setErrorAndRollback(index - applied, new Status(RaftError.ESTATEMACHINE,
+                "StateMachine meet critical error: %s.", t.getMessage()));
+        } finally {
+            // metrics: qps
+            this.applyMeter.mark(applied);
+        }
+    }
+
+    private int batchApplyAndRecycle(final byte opByte, final KVStateOutputList kvStates) {
+        try {
+            final int size = kvStates.size();
+
+            if (size == 0) {
+                return 0;
+            }
+
+            if (!KVOperation.isValidOp(opByte)) {
+                throw new IllegalKVOperationException("Unknown operation: " + opByte);
+            }
+
+            // metrics: op qps
+            final Meter opApplyMeter = KVMetrics.meter(STATE_MACHINE_APPLY_QPS, String.valueOf(this.region.getId()),
+                KVOperation.opName(opByte));
+            opApplyMeter.mark(size);
+            this.batchWriteHistogram.update(size);
+
+            // do batch apply
+            batchApply(opByte, kvStates);
+
+            return size;
+        } finally {
+            RecycleUtil.recycle(kvStates);
+        }
+    }
+
+    private void batchApply(final byte opType, final KVStateOutputList kvStates) {
+        switch (opType) {
+            case KVOperation.PUT:
+                this.rawKVStore.batchPut(kvStates);
+                break;
+            case KVOperation.PUT_IF_ABSENT:
+                this.rawKVStore.batchPutIfAbsent(kvStates);
+                break;
+            case KVOperation.PUT_LIST:
+                this.rawKVStore.batchPutList(kvStates);
+                break;
+            case KVOperation.DELETE:
+                this.rawKVStore.batchDelete(kvStates);
+                break;
+            case KVOperation.DELETE_RANGE:
+                this.rawKVStore.batchDeleteRange(kvStates);
+                break;
+            case KVOperation.DELETE_LIST:
+                this.rawKVStore.batchDeleteList(kvStates);
+                break;
+            case KVOperation.GET_SEQUENCE:
+                this.rawKVStore.batchGetSequence(kvStates);
+                break;
+            case KVOperation.NODE_EXECUTE:
+                this.rawKVStore.batchNodeExecute(kvStates, isLeader());
+                break;
+            case KVOperation.KEY_LOCK:
+                this.rawKVStore.batchTryLockWith(kvStates);
+                break;
+            case KVOperation.KEY_LOCK_RELEASE:
+                this.rawKVStore.batchReleaseLockWith(kvStates);
+                break;
+            case KVOperation.GET:
+                this.rawKVStore.batchGet(kvStates);
+                break;
+            case KVOperation.MULTI_GET:
+                this.rawKVStore.batchMultiGet(kvStates);
+                break;
+            case KVOperation.CONTAINS_KEY:
+                this.rawKVStore.batchContainsKey(kvStates);
+                break;
+            case KVOperation.SCAN:
+                this.rawKVStore.batchScan(kvStates);
+                break;
+            case KVOperation.REVERSE_SCAN:
+                this.rawKVStore.batchReverseScan(kvStates);
+                break;
+            case KVOperation.GET_PUT:
+                this.rawKVStore.batchGetAndPut(kvStates);
+                break;
+            case KVOperation.COMPARE_PUT:
+                this.rawKVStore.batchCompareAndPut(kvStates);
+                break;
+            case KVOperation.COMPARE_PUT_ALL:
+                this.rawKVStore.batchCompareAndPutAll(kvStates);
+                break;
+            case KVOperation.MERGE:
+                this.rawKVStore.batchMerge(kvStates);
+                break;
+            case KVOperation.RESET_SEQUENCE:
+                this.rawKVStore.batchResetSequence(kvStates);
+                break;
+            case KVOperation.RANGE_SPLIT:
+                doSplit(kvStates);
+                break;
+            default:
+                throw new IllegalKVOperationException("Unknown operation: " + opType);
+        }
+    }
+
+    private void doSplit(final KVStateOutputList kvStates) {
+        final byte[] parentKey = this.region.getStartKey();
+        for (final KVState kvState : kvStates) {
+            final KVOperation op = kvState.getOp();
+            final long currentRegionId = op.getCurrentRegionId();
+            final long newRegionId = op.getNewRegionId();
+            final byte[] splitKey = op.getKey();
+            final KVStoreClosure closure = kvState.getDone();
+            try {
+                this.rawKVStore.initFencingToken(parentKey, splitKey);
+                this.storeEngine.doSplit(currentRegionId, newRegionId, splitKey);
+                if (closure != null) {
+                    // null on follower
+                    closure.setData(Boolean.TRUE);
+                    closure.run(Status.OK());
+                }
+            } catch (final Throwable t) {
+                LOG.error("Fail to split, regionId={}, newRegionId={}, splitKey={}.", currentRegionId, newRegionId,
+                    BytesUtil.toHex(splitKey));
+                setCriticalError(closure, t);
+            }
+        }
+    }
+
+    @Override
+    public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+        this.storeSnapshotFile.save(writer, this.region.copy(), done, this.storeEngine.getSnapshotExecutor());
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        if (isLeader()) {
+            LOG.warn("Leader is not supposed to load snapshot.");
+            return false;
+        }
+        return this.storeSnapshotFile.load(reader, this.region.copy());
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        super.onLeaderStart(term);
+        this.leaderTerm.set(term);
+        // Because of the raft state machine must be a sequential commit, in order to prevent the user
+        // doing something (needs to go through the raft state machine) in the listeners, we need
+        // asynchronously triggers the listeners.
+        final List listeners = this.storeEngine.getStateListenerContainer() //
+            .getStateListenerGroup(getRegionId());
+        if (listeners.isEmpty()) {
+            return;
+        }
+        this.storeEngine.getRaftStateTrigger().execute(() -> {
+            for (final StateListener listener : listeners) { // iterator the snapshot
+                listener.onLeaderStart(term);
+            }
+        });
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        super.onLeaderStop(status);
+        final long oldTerm = this.leaderTerm.get();
+        this.leaderTerm.set(-1L);
+        // Because of the raft state machine must be a sequential commit, in order to prevent the user
+        // doing something (needs to go through the raft state machine) in the listeners, we asynchronously
+        // triggers the listeners.
+        final List listeners = this.storeEngine.getStateListenerContainer() //
+            .getStateListenerGroup(getRegionId());
+        if (listeners.isEmpty()) {
+            return;
+        }
+        this.storeEngine.getRaftStateTrigger().execute(() -> {
+            for (final StateListener listener : listeners) { // iterator the snapshot
+                listener.onLeaderStop(oldTerm);
+            }
+        });
+    }
+
+    @Override
+    public void onStartFollowing(final LeaderChangeContext ctx) {
+        super.onStartFollowing(ctx);
+        // Because of the raft state machine must be a sequential commit, in order to prevent the user
+        // doing something (needs to go through the raft state machine) in the listeners, we need
+        // asynchronously triggers the listeners.
+        final List listeners = this.storeEngine.getStateListenerContainer() //
+            .getStateListenerGroup(getRegionId());
+        if (listeners.isEmpty()) {
+            return;
+        }
+        this.storeEngine.getRaftStateTrigger().execute(() -> {
+            for (final StateListener listener : listeners) { // iterator the snapshot
+                listener.onStartFollowing(ctx.getLeaderId(), ctx.getTerm());
+            }
+        });
+    }
+
+    @Override
+    public void onStopFollowing(final LeaderChangeContext ctx) {
+        super.onStopFollowing(ctx);
+        // Because of the raft state machine must be a sequential commit, in order to prevent the user
+        // doing something (needs to go through the raft state machine) in the listeners, we need
+        // asynchronously triggers the listeners.
+        final List listeners = this.storeEngine.getStateListenerContainer() //
+            .getStateListenerGroup(getRegionId());
+        if (listeners.isEmpty()) {
+            return;
+        }
+        this.storeEngine.getRaftStateTrigger().execute(() -> {
+            for (final StateListener listener : listeners) { // iterator the snapshot
+                listener.onStopFollowing(ctx.getLeaderId(), ctx.getTerm());
+            }
+        });
+    }
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    public long getRegionId() {
+        return this.region.getId();
+    }
+
+    /**
+     * Sets critical error and halt the state machine.
+     *
+     * If current node is a leader, first reply to client
+     * failure response.
+     *
+     * @param closure callback
+     * @param ex      critical error
+     */
+    private static void setCriticalError(final KVStoreClosure closure, final Throwable ex) {
+        // Will call closure#run in FSMCaller
+        if (closure != null) {
+            closure.setError(Errors.forException(ex));
+        }
+        ThrowUtil.throwException(ex);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/LongSequence.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/LongSequence.java
new file mode 100644
index 0000000..6d7bf17
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/LongSequence.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class LongSequence {
+
+    private final long base;
+
+    private Sequence   sequence;
+    private long       value;
+
+    public LongSequence() {
+        this(0);
+    }
+
+    public LongSequence(long base) {
+        this.base = base;
+    }
+
+    public synchronized long get() {
+        return this.base + this.value;
+    }
+
+    public synchronized long next() {
+        if (this.sequence == null) {
+            reset();
+        }
+        final long val = this.value++;
+        if (val == this.sequence.getEndValue()) {
+            reset();
+            return this.base + this.value++;
+        }
+        return this.base + val;
+    }
+
+    private void reset() {
+        this.sequence = getNextSequence();
+        this.value = sequence.getStartValue();
+    }
+
+    public abstract Sequence getNextSequence();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVIterator.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVIterator.java
new file mode 100644
index 0000000..546b931
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVIterator.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentNavigableMap;
+
+/**
+ * @author jiachun.fjc
+ */
+public class MemoryKVIterator implements KVIterator {
+
+    private final ConcurrentNavigableMap db;
+
+    private Map.Entry                    cursorEntry;
+
+    public MemoryKVIterator(ConcurrentNavigableMap db) {
+        this.db = db;
+    }
+
+    @Override
+    public boolean isValid() {
+        return this.cursorEntry != null;
+    }
+
+    @Override
+    public void seekToFirst() {
+        this.cursorEntry = this.db.firstEntry();
+    }
+
+    @Override
+    public void seekToLast() {
+        this.cursorEntry = this.db.lastEntry();
+    }
+
+    @Override
+    public void seek(final byte[] target) {
+        this.cursorEntry = this.db.ceilingEntry(target);
+    }
+
+    @Override
+    public void seekForPrev(final byte[] target) {
+        this.cursorEntry = this.db.lowerEntry(target);
+    }
+
+    @Override
+    public void next() {
+        this.cursorEntry = this.db.higherEntry(this.cursorEntry.getKey());
+    }
+
+    @Override
+    public void prev() {
+        this.cursorEntry = this.db.lowerEntry(this.cursorEntry.getKey());
+    }
+
+    @Override
+    public byte[] key() {
+        return this.cursorEntry.getKey();
+    }
+
+    @Override
+    public byte[] value() {
+        return this.cursorEntry.getValue();
+    }
+
+    @Override
+    public void close() throws Exception {
+        // no-op
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVStoreSnapshotFile.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVStoreSnapshotFile.java
new file mode 100644
index 0000000..be9ef94
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryKVStoreSnapshotFile.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+import com.alipay.sofa.jraft.rhea.errors.StorageException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.RegionHelper;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.Bits;
+
+import static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MemoryKVStoreSnapshotFile extends AbstractKVStoreSnapshotFile {
+
+    private final MemoryRawKVStore kvStore;
+
+    MemoryKVStoreSnapshotFile(MemoryRawKVStore kvStore) {
+        this.kvStore = kvStore;
+    }
+
+    @Override
+    CompletableFuture doSnapshotSave(final String snapshotPath, final Region region,
+                                                            final ExecutorService executor) throws Exception {
+        this.kvStore.doSnapshotSave(this, snapshotPath, region);
+        return CompletableFuture.completedFuture(writeMetadata(region));
+    }
+
+    @Override
+    void doSnapshotLoad(final String snapshotPath, final LocalFileMeta meta, final Region region) throws Exception {
+        final File file = new File(snapshotPath);
+        if (!file.exists()) {
+            throw new StorageException("Snapshot file [" + snapshotPath + "] not exists");
+        }
+        final Region snapshotRegion = readMetadata(meta, Region.class);
+        if (!RegionHelper.isSameRange(region, snapshotRegion)) {
+            throw new StorageException("Invalid snapshot region: " + snapshotRegion + ", current region is: " + region);
+        }
+        this.kvStore.doSnapshotLoad(this, snapshotPath);
+    }
+
+     void writeToFile(final String rootPath, final String fileName, final Persistence persist) throws Exception {
+        final Path path = Paths.get(rootPath, fileName);
+        try (final FileOutputStream out = new FileOutputStream(path.toFile());
+                final BufferedOutputStream bufOutput = new BufferedOutputStream(out)) {
+            final byte[] bytes = this.serializer.writeObject(persist);
+            final byte[] lenBytes = new byte[4];
+            Bits.putInt(lenBytes, 0, bytes.length);
+            bufOutput.write(lenBytes);
+            bufOutput.write(bytes);
+            bufOutput.flush();
+            out.getFD().sync();
+        }
+    }
+
+     T readFromFile(final String rootPath, final String fileName, final Class clazz) throws Exception {
+        final Path path = Paths.get(rootPath, fileName);
+        final File file = path.toFile();
+        if (!file.exists()) {
+            throw new NoSuchFieldException(path.toString());
+        }
+        try (final FileInputStream in = new FileInputStream(file);
+                final BufferedInputStream bufInput = new BufferedInputStream(in)) {
+            final byte[] lenBytes = new byte[4];
+            int read = bufInput.read(lenBytes);
+            if (read != lenBytes.length) {
+                throw new IOException("fail to read snapshot file length, expects " + lenBytes.length
+                                      + " bytes, but read " + read);
+            }
+            final int len = Bits.getInt(lenBytes, 0);
+            final byte[] bytes = new byte[len];
+            read = bufInput.read(bytes);
+            if (read != bytes.length) {
+                throw new IOException("fail to read snapshot file, expects " + bytes.length + " bytes, but read "
+                                      + read);
+            }
+            return this.serializer.readObject(bytes, clazz);
+        }
+    }
+
+    static class Persistence {
+
+        private final T data;
+
+        public Persistence(T data) {
+            this.data = data;
+        }
+
+        public T data() {
+            return data;
+        }
+    }
+
+    /**
+     * The data of sequences
+     */
+    static class SequenceDB extends Persistence> {
+
+        public SequenceDB(Map data) {
+            super(data);
+        }
+    }
+
+    /**
+     * The data of fencing token keys
+     */
+    static class FencingKeyDB extends Persistence> {
+
+        public FencingKeyDB(Map data) {
+            super(data);
+        }
+    }
+
+    /**
+     * The data of lock info
+     */
+    static class LockerDB extends Persistence> {
+
+        public LockerDB(Map data) {
+            super(data);
+        }
+    }
+
+    /**
+     * The data will be cut into many small portions, each called a segment
+     */
+    static class Segment extends Persistence>> {
+
+        public Segment(List> data) {
+            super(data);
+        }
+    }
+
+    /**
+     * The 'tailIndex' records the largest segment number (the segment number starts from 0)
+     */
+    static class TailIndex extends Persistence {
+
+        public TailIndex(Integer data) {
+            super(data);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryRawKVStore.java
new file mode 100644
index 0000000..97c0abb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MemoryRawKVStore.java
@@ -0,0 +1,828 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentNavigableMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.MemoryKVStoreSnapshotFile.SequenceDB;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.RegionHelper;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.codahale.metrics.Timer;
+
+import static com.alipay.sofa.jraft.rhea.storage.MemoryKVStoreSnapshotFile.FencingKeyDB;
+import static com.alipay.sofa.jraft.rhea.storage.MemoryKVStoreSnapshotFile.LockerDB;
+import static com.alipay.sofa.jraft.rhea.storage.MemoryKVStoreSnapshotFile.Segment;
+import static com.alipay.sofa.jraft.rhea.storage.MemoryKVStoreSnapshotFile.TailIndex;
+
+/**
+ * @author jiachun.fjc
+ */
+public class MemoryRawKVStore extends BatchRawKVStore {
+
+    private static final Logger                          LOG            = LoggerFactory
+                                                                            .getLogger(MemoryRawKVStore.class);
+
+    private static final String                          SEQUENCE_DB    = "sequenceDB";
+    private static final String                          FENCING_KEY_DB = "fencingKeyDB";
+    private static final String                          LOCKER_DB      = "lockerDB";
+    private static final String                          SEGMENT        = "segment";
+    private static final String                          TAIL_INDEX     = "tailIndex";
+
+    private static final byte                            DELIMITER      = (byte) ',';
+    private static final Comparator              COMPARATOR     = BytesUtil.getDefaultByteArrayComparator();
+
+    private final ConcurrentNavigableMap defaultDB      = new ConcurrentSkipListMap<>(COMPARATOR);
+    private final Map                   sequenceDB     = new ConcurrentHashMap<>();
+    private final Map                   fencingKeyDB   = new ConcurrentHashMap<>();
+    private final Map  lockerDB       = new ConcurrentHashMap<>();
+
+    private volatile MemoryDBOptions                     opts;
+
+    @Override
+    public boolean init(final MemoryDBOptions opts) {
+        this.opts = opts;
+        LOG.info("[MemoryRawKVStore] start successfully, options: {}.", opts);
+        return true;
+    }
+
+    @Override
+    public void shutdown() {
+        this.defaultDB.clear();
+        this.sequenceDB.clear();
+        this.fencingKeyDB.clear();
+        this.lockerDB.clear();
+    }
+
+    @Override
+    public KVIterator localIterator() {
+        return new MemoryKVIterator(this.defaultDB);
+    }
+
+    @Override
+    public void get(final byte[] key, @SuppressWarnings("unused") final boolean readOnlySafe,
+                    final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET");
+        try {
+            final byte[] value = this.defaultDB.get(key);
+            setSuccess(closure, value);
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET], key: [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [GET]");
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void multiGet(final List keys, @SuppressWarnings("unused") final boolean readOnlySafe,
+                         final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("MULTI_GET");
+        try {
+            final Map resultMap = Maps.newHashMap();
+            for (final byte[] key : keys) {
+                final byte[] value = this.defaultDB.get(key);
+                if (value == null) {
+                    continue;
+                }
+                resultMap.put(ByteArray.wrap(key), value);
+            }
+            setSuccess(closure, resultMap);
+        } catch (final Exception e) {
+            LOG.error("Fail to [MULTI_GET], key size: [{}], {}.", keys.size(), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [MULTI_GET]");
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void containsKey(final byte[] key, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("CONTAINS_KEY");
+        try {
+            final boolean exists = this.defaultDB.containsKey(key);
+            setSuccess(closure, exists);
+        } catch (final Exception e) {
+            LOG.error("Fail to [CONTAINS_KEY], key: [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [CONTAINS_KEY]");
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit,
+                     @SuppressWarnings("unused") final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("SCAN");
+        final List entries = Lists.newArrayList();
+        final int maxCount = normalizeLimit(limit);
+        final ConcurrentNavigableMap subMap;
+        final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+        if (endKey == null) {
+            subMap = this.defaultDB.tailMap(realStartKey);
+        } else {
+            subMap = this.defaultDB.subMap(realStartKey, endKey);
+        }
+        try {
+            for (final Map.Entry entry : subMap.entrySet()) {
+                entries.add(new KVEntry(entry.getKey(), returnValue ? entry.getValue() : null));
+                if (entries.size() >= maxCount) {
+                    break;
+                }
+            }
+            setSuccess(closure, entries);
+        } catch (final Exception e) {
+            LOG.error("Fail to [SCAN], range: ['[{}, {})'], {}.", BytesUtil.toHex(startKey), BytesUtil.toHex(endKey),
+                StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [SCAN]");
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit,
+                            @SuppressWarnings("unused") final boolean readOnlySafe, final boolean returnValue,
+                            final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("REVERSE_SCAN");
+        final List entries = Lists.newArrayList();
+        final int maxCount = normalizeLimit(limit);
+        final ConcurrentNavigableMap subMap;
+        final byte[] realEndKey = BytesUtil.nullToEmpty(endKey);
+        if (startKey == null) {
+            subMap = this.defaultDB.descendingMap().headMap(realEndKey);
+        } else {
+            subMap = this.defaultDB.descendingMap().subMap(startKey, realEndKey);
+        }
+        try {
+            for (final Map.Entry entry : subMap.entrySet()) {
+                entries.add(new KVEntry(entry.getKey(), returnValue ? entry.getValue() : null));
+                if (entries.size() >= maxCount) {
+                    break;
+                }
+            }
+            setSuccess(closure, entries);
+        } catch (final Exception e) {
+            LOG.error("Fail to [REVERSE_SCAN], range: ['[{}, {})'], {}.", BytesUtil.toHex(startKey),
+                BytesUtil.toHex(endKey), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [REVERSE_SCAN]");
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void getSequence(final byte[] seqKey, final int step, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET_SEQUENCE");
+        try {
+            final ByteArray wrappedKey = ByteArray.wrap(seqKey);
+            Long startVal = this.sequenceDB.get(wrappedKey);
+            startVal = startVal == null ? 0 : startVal;
+            if (step < 0) {
+                // never get here
+                setFailure(closure, "Fail to [GET_SEQUENCE], step must >= 0");
+                return;
+            }
+            if (step == 0) {
+                setSuccess(closure, new Sequence(startVal, startVal));
+                return;
+            }
+            final long endVal = getSafeEndValueForSequence(startVal, step);
+            if (startVal != endVal) {
+                this.sequenceDB.put(wrappedKey, endVal);
+            }
+            setSuccess(closure, new Sequence(startVal, endVal));
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET_SEQUENCE], [key = {}, step = {}], {}.", BytesUtil.toHex(seqKey), step,
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [GET_SEQUENCE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void resetSequence(final byte[] seqKey, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("RESET_SEQUENCE");
+        try {
+            this.sequenceDB.remove(ByteArray.wrap(seqKey));
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [RESET_SEQUENCE], [key = {}], {}.", BytesUtil.toHex(seqKey),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [RESET_SEQUENCE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void put(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT");
+        try {
+            this.defaultDB.put(key, value);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [PUT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void getAndPut(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET_PUT");
+        try {
+            final byte[] prevVal = this.defaultDB.put(key, value);
+            setSuccess(closure, prevVal);
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET_PUT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [GET_PUT]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void compareAndPut(final byte[] key, final byte[] expect, final byte[] update, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("COMPARE_PUT");
+        try {
+            final byte[] actual = this.defaultDB.get(key);
+            if (Arrays.equals(expect, actual)) {
+                this.defaultDB.put(key, update);
+                setSuccess(closure, Boolean.TRUE);
+            } else {
+                setSuccess(closure, Boolean.FALSE);
+            }
+        } catch (final Exception e) {
+            LOG.error("Fail to [COMPARE_PUT], [{}, {}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(expect),
+                BytesUtil.toHex(update), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [COMPARE_PUT]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void merge(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("MERGE");
+        try {
+            this.defaultDB.compute(key, (ignored, oldVal) -> {
+                if (oldVal == null) {
+                    return value;
+                } else {
+                    final byte[] newVal = new byte[oldVal.length + 1 + value.length];
+                    System.arraycopy(oldVal, 0, newVal, 0, oldVal.length);
+                    newVal[oldVal.length] = DELIMITER;
+                    System.arraycopy(value, 0, newVal, oldVal.length + 1, value.length);
+                    return newVal;
+                }
+            });
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [MERGE], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                    StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [MERGE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void put(final List entries, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT_LIST");
+        try {
+            for (final KVEntry entry : entries) {
+                this.defaultDB.put(entry.getKey(), entry.getValue());
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [PUT_LIST], [size = {}], {}.", entries.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT_LIST]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void compareAndPutAll(final List entries, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("COMPARE_PUT_ALL");
+        try {
+            for (final CASEntry entry : entries) {
+                final byte[] actual = this.defaultDB.get(entry.getKey());
+                if (!Arrays.equals(entry.getExpect(), actual)) {
+                    setSuccess(closure, Boolean.FALSE);
+                    return;
+                }
+            }
+
+            for (final CASEntry entry : entries) {
+                this.defaultDB.put(entry.getKey(), entry.getUpdate());
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [COMPARE_PUT_ALL], [size = {}], {}.", entries.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [COMPARE_PUT_ALL]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void putIfAbsent(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT_IF_ABSENT");
+        try {
+            final byte[] prevValue = this.defaultDB.putIfAbsent(key, value);
+            setSuccess(closure, prevValue);
+        } catch (final Exception e) {
+            LOG.error("Fail to [PUT_IF_ABSENT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT_IF_ABSENT]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void tryLockWith(final byte[] key, final byte[] fencingKey, final boolean keepLease,
+                            final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("TRY_LOCK");
+        try {
+            // The algorithm relies on the assumption that while there is no
+            // synchronized clock across the processes, still the local time in
+            // every process flows approximately at the same rate, with an error
+            // which is small compared to the auto-release time of the lock.
+            final long now = acquirer.getLockingTimestamp();
+            final long timeoutMillis = acquirer.getLeaseMillis();
+            final ByteArray wrappedKey = ByteArray.wrap(key);
+            final DistributedLock.Owner prevOwner = this.lockerDB.get(wrappedKey);
+
+            final DistributedLock.Owner owner;
+            // noinspection ConstantConditions
+            do {
+                final DistributedLock.OwnerBuilder builder = DistributedLock.newOwnerBuilder();
+                if (prevOwner == null) {
+                    // no others own this lock
+                    if (keepLease) {
+                        // it wants to keep the lease but too late, will return failure
+                        owner = builder //
+                            // set acquirer id
+                            .id(acquirer.getId())
+                            // fail to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_FAIL)
+                            // set failure
+                            .success(false).build();
+                        break;
+                    }
+                    // is first time to try lock (another possibility is that this lock has been deleted),
+                    // will return successful
+                    owner = builder //
+                        // set acquirer id, now it will own the lock
+                        .id(acquirer.getId())
+                        // set a new deadline
+                        .deadlineMillis(now + timeoutMillis)
+                        // first time to acquire and success
+                        .remainingMillis(DistributedLock.OwnerBuilder.FIRST_TIME_SUCCESS)
+                        // create a new fencing token
+                        .fencingToken(getNextFencingToken(fencingKey))
+                        // init acquires
+                        .acquires(1)
+                        // set acquirer ctx
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.lockerDB.put(wrappedKey, owner);
+                    break;
+                }
+
+                // this lock has an owner, check if it has expired
+                final long remainingMillis = prevOwner.getDeadlineMillis() - now;
+                if (remainingMillis < 0) {
+                    // the previous owner is out of lease
+                    if (keepLease) {
+                        // it wants to keep the lease but too late, will return failure
+                        owner = builder //
+                            // still previous owner id
+                            .id(prevOwner.getId())
+                            // do not update
+                            .deadlineMillis(prevOwner.getDeadlineMillis())
+                            // fail to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_FAIL)
+                            // set previous ctx
+                            .context(prevOwner.getContext())
+                            // set failure
+                            .success(false).build();
+                        break;
+                    }
+                    // create new lock owner
+                    owner = builder //
+                        // set acquirer id, now it will own the lock
+                        .id(acquirer.getId())
+                        // set a new deadline
+                        .deadlineMillis(now + timeoutMillis)
+                        // success as a new acquirer
+                        .remainingMillis(DistributedLock.OwnerBuilder.NEW_ACQUIRE_SUCCESS)
+                        // create a new fencing token
+                        .fencingToken(getNextFencingToken(fencingKey))
+                        // init acquires
+                        .acquires(1)
+                        // set acquirer ctx
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.lockerDB.put(wrappedKey, owner);
+                    break;
+                }
+
+                // the previous owner is not out of lease (remainingMillis >= 0)
+                final boolean isReentrant = prevOwner.isSameAcquirer(acquirer);
+                if (isReentrant) {
+                    // is the same old friend come back (reentrant lock)
+                    if (keepLease) {
+                        // the old friend only wants to keep lease of lock
+                        owner = builder //
+                            // still previous owner id
+                            .id(prevOwner.getId())
+                            // update the deadline to keep lease
+                            .deadlineMillis(now + timeoutMillis)
+                            // success to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_SUCCESS)
+                            // keep fencing token
+                            .fencingToken(prevOwner.getFencingToken())
+                            // keep acquires
+                            .acquires(prevOwner.getAcquires())
+                            // do not update ctx when keeping lease
+                            .context(prevOwner.getContext())
+                            // set successful
+                            .success(true).build();
+                        this.lockerDB.put(wrappedKey, owner);
+                        break;
+                    }
+                    // now we are sure that is an old friend who is back again (reentrant lock)
+                    owner = builder //
+                        // still previous owner id
+                        .id(prevOwner.getId())
+                        // by the way, the lease will also be kept
+                        .deadlineMillis(now + timeoutMillis)
+                        // success reentrant
+                        .remainingMillis(DistributedLock.OwnerBuilder.REENTRANT_SUCCESS)
+                        // keep fencing token
+                        .fencingToken(prevOwner.getFencingToken())
+                        // acquires++
+                        .acquires(prevOwner.getAcquires() + 1)
+                        // update ctx when reentrant
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.lockerDB.put(wrappedKey, owner);
+                    break;
+                }
+
+                // the lock is exist and also prev locker is not the same as current
+                owner = builder //
+                    // set previous owner id to tell who is the real owner
+                    .id(prevOwner.getId())
+                    // set the remaining lease time of current owner
+                    .remainingMillis(remainingMillis)
+                    // set previous ctx
+                    .context(prevOwner.getContext())
+                    // set failure
+                    .success(false).build();
+                LOG.debug("Another locker [{}] is trying the existed lock [{}].", acquirer, prevOwner);
+            } while (false);
+
+            setSuccess(closure, owner);
+        } catch (final Exception e) {
+            LOG.error("Fail to [TRY_LOCK], [{}, {}], {}.", BytesUtil.toHex(key), acquirer, StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [TRY_LOCK]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void releaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("RELEASE_LOCK");
+        try {
+            final ByteArray wrappedKey = ByteArray.wrap(key);
+            final DistributedLock.Owner prevOwner = this.lockerDB.get(wrappedKey);
+
+            final DistributedLock.Owner owner;
+            // noinspection ConstantConditions
+            do {
+                final DistributedLock.OwnerBuilder builder = DistributedLock.newOwnerBuilder();
+                if (prevOwner == null) {
+                    LOG.warn("Lock not exist: {}.", acquirer);
+                    owner = builder //
+                        // set acquirer id
+                        .id(acquirer.getId())
+                        // set acquirer fencing token
+                        .fencingToken(acquirer.getFencingToken())
+                        // set acquires=0
+                        .acquires(0)
+                        // set successful
+                        .success(true).build();
+                    break;
+                }
+
+                if (prevOwner.isSameAcquirer(acquirer)) {
+                    final long acquires = prevOwner.getAcquires() - 1;
+                    owner = builder //
+                        // still previous owner id
+                        .id(prevOwner.getId())
+                        // do not update deadline
+                        .deadlineMillis(prevOwner.getDeadlineMillis())
+                        // keep fencing token
+                        .fencingToken(prevOwner.getFencingToken())
+                        // acquires--
+                        .acquires(acquires)
+                        // set previous ctx
+                        .context(prevOwner.getContext())
+                        // set successful
+                        .success(true).build();
+                    if (acquires <= 0) {
+                        // real delete, goodbye ~
+                        this.lockerDB.remove(wrappedKey);
+                    } else {
+                        // acquires--
+                        this.lockerDB.put(wrappedKey, owner);
+                    }
+                    break;
+                }
+
+                // invalid acquirer, can't to release the lock
+                owner = builder //
+                    // set previous owner id to tell who is the real owner
+                    .id(prevOwner.getId())
+                    // keep previous fencing token
+                    .fencingToken(prevOwner.getFencingToken())
+                    // do not update acquires
+                    .acquires(prevOwner.getAcquires())
+                    // set previous ctx
+                    .context(prevOwner.getContext())
+                    // set failure
+                    .success(false).build();
+                LOG.warn("The lock owner is: [{}], [{}] could't release it.", prevOwner, acquirer);
+            } while (false);
+
+            setSuccess(closure, owner);
+        } catch (final Exception e) {
+            LOG.error("Fail to [RELEASE_LOCK], [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [RELEASE_LOCK]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    private long getNextFencingToken(final byte[] fencingKey) {
+        final Timer.Context timeCtx = getTimeContext("FENCING_TOKEN");
+        try {
+            // Don't worry about the token number overflow.
+            // It takes about 290,000 years for the 1 million TPS system
+            // to use the numbers in the range [0 ~ Long.MAX_VALUE].
+            final byte[] realKey = BytesUtil.nullToEmpty(fencingKey);
+            return this.fencingKeyDB.compute(ByteArray.wrap(realKey), (key, prevVal) -> {
+                if (prevVal == null) {
+                    return 1L;
+                }
+                return ++prevVal;
+            });
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void delete(final byte[] key, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE");
+        try {
+            this.defaultDB.remove(key);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [DELETE], [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void deleteRange(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE_RANGE");
+        try {
+            final ConcurrentNavigableMap subMap = this.defaultDB.subMap(startKey, endKey);
+            if (!subMap.isEmpty()) {
+                subMap.clear();
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [DELETE_RANGE], ['[{}, {})'], {}.", BytesUtil.toHex(startKey), BytesUtil.toHex(endKey),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE_RANGE]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void delete(final List keys, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE_LIST");
+        try {
+            for (final byte[] key : keys) {
+                this.defaultDB.remove(key);
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [DELETE_LIST], [size = {}], {}.", keys.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE_LIST]", e);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public long getApproximateKeysInRange(final byte[] startKey, final byte[] endKey) {
+        final Timer.Context timeCtx = getTimeContext("APPROXIMATE_KEYS");
+        try {
+            final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+            final ConcurrentNavigableMap subMap;
+            if (endKey == null) {
+                subMap = this.defaultDB.tailMap(realStartKey);
+            } else {
+                subMap = this.defaultDB.subMap(realStartKey, endKey);
+            }
+            return subMap.size();
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public byte[] jumpOver(final byte[] startKey, final long distance) {
+        final Timer.Context timeCtx = getTimeContext("JUMP_OVER");
+        try {
+            final byte[] realStartKey = BytesUtil.nullToEmpty(startKey);
+            final ConcurrentNavigableMap tailMap = this.defaultDB.tailMap(realStartKey);
+            if (tailMap.isEmpty()) {
+                return null;
+            }
+            long approximateKeys = 0;
+            byte[] lastKey = null;
+            for (final byte[] key : tailMap.keySet()) {
+                lastKey = key;
+                if (++approximateKeys >= distance) {
+                    break;
+                }
+            }
+            if (lastKey == null) {
+                return null;
+            }
+            final byte[] endKey = new byte[lastKey.length];
+            System.arraycopy(lastKey, 0, endKey, 0, lastKey.length);
+            return endKey;
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void initFencingToken(final byte[] parentKey, final byte[] childKey) {
+        final Timer.Context timeCtx = getTimeContext("INIT_FENCING_TOKEN");
+        try {
+            final byte[] realKey = BytesUtil.nullToEmpty(parentKey);
+            final Long parentVal = this.fencingKeyDB.get(ByteArray.wrap(realKey));
+            if (parentVal == null) {
+                return;
+            }
+            this.fencingKeyDB.put(ByteArray.wrap(childKey), parentVal);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    void doSnapshotSave(final MemoryKVStoreSnapshotFile snapshotFile, final String snapshotPath, final Region region)
+                                                                                                                     throws Exception {
+        final Timer.Context timeCtx = getTimeContext("SNAPSHOT_SAVE");
+        try {
+            final String tempPath = snapshotPath + "_temp";
+            final File tempFile = new File(tempPath);
+            FileUtils.deleteDirectory(tempFile);
+            FileUtils.forceMkdir(tempFile);
+
+            snapshotFile.writeToFile(tempPath, SEQUENCE_DB, new SequenceDB(subRangeMap(this.sequenceDB, region)));
+            snapshotFile
+                .writeToFile(tempPath, FENCING_KEY_DB, new FencingKeyDB(subRangeMap(this.fencingKeyDB, region)));
+            snapshotFile.writeToFile(tempPath, LOCKER_DB, new LockerDB(subRangeMap(this.lockerDB, region)));
+            final int size = this.opts.getKeysPerSegment();
+            final List> segment = Lists.newArrayListWithCapacity(size);
+            int index = 0;
+            final byte[] realStartKey = BytesUtil.nullToEmpty(region.getStartKey());
+            final byte[] endKey = region.getEndKey();
+            final NavigableMap subMap;
+            if (endKey == null) {
+                subMap = this.defaultDB.tailMap(realStartKey);
+            } else {
+                subMap = this.defaultDB.subMap(realStartKey, endKey);
+            }
+            for (final Map.Entry entry : subMap.entrySet()) {
+                segment.add(Pair.of(entry.getKey(), entry.getValue()));
+                if (segment.size() >= size) {
+                    snapshotFile.writeToFile(tempPath, SEGMENT + index++, new Segment(segment));
+                    segment.clear();
+                }
+            }
+            if (!segment.isEmpty()) {
+                snapshotFile.writeToFile(tempPath, SEGMENT + index++, new Segment(segment));
+                segment.clear();
+            }
+            snapshotFile.writeToFile(tempPath, TAIL_INDEX, new TailIndex(--index));
+
+            final File destinationPath = new File(snapshotPath);
+            FileUtils.deleteDirectory(destinationPath);
+            FileUtils.moveDirectory(tempFile, destinationPath);
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    void doSnapshotLoad(final MemoryKVStoreSnapshotFile snapshotFile, final String snapshotPath) throws Exception {
+        final Timer.Context timeCtx = getTimeContext("SNAPSHOT_LOAD");
+        try {
+            final SequenceDB sequenceDB = snapshotFile.readFromFile(snapshotPath, SEQUENCE_DB, SequenceDB.class);
+            final FencingKeyDB fencingKeyDB = snapshotFile.readFromFile(snapshotPath, FENCING_KEY_DB,
+                FencingKeyDB.class);
+            final LockerDB lockerDB = snapshotFile.readFromFile(snapshotPath, LOCKER_DB, LockerDB.class);
+
+            this.sequenceDB.putAll(sequenceDB.data());
+            this.fencingKeyDB.putAll(fencingKeyDB.data());
+            this.lockerDB.putAll(lockerDB.data());
+
+            final TailIndex tailIndex = snapshotFile.readFromFile(snapshotPath, TAIL_INDEX, TailIndex.class);
+            final int tail = tailIndex.data();
+            final List segments = Lists.newArrayListWithCapacity(tail + 1);
+            for (int i = 0; i <= tail; i++) {
+                final Segment segment = snapshotFile.readFromFile(snapshotPath, SEGMENT + i, Segment.class);
+                segments.add(segment);
+            }
+            for (final Segment segment : segments) {
+                for (final Pair p : segment.data()) {
+                    this.defaultDB.put(p.getKey(), p.getValue());
+                }
+            }
+        } finally {
+            timeCtx.stop();
+        }
+    }
+
+    static  Map subRangeMap(final Map input, final Region region) {
+        if (RegionHelper.isSingleGroup(region)) {
+            return input;
+        }
+        final Map output = new HashMap<>();
+        for (final Map.Entry entry : input.entrySet()) {
+            final ByteArray key = entry.getKey();
+            if (RegionHelper.isKeyInRegion(key.getBytes(), region)) {
+                output.put(key, entry.getValue());
+            }
+        }
+        return output;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsKVClosureAdapter.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsKVClosureAdapter.java
new file mode 100644
index 0000000..5f4fbcb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsKVClosureAdapter.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.List;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.codahale.metrics.Timer;
+
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.REGION_BYTES_READ;
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.REGION_BYTES_WRITTEN;
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.REGION_KEYS_READ;
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.REGION_KEYS_WRITTEN;
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.RPC_REQUEST_HANDLE_TIMER;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MetricsKVClosureAdapter implements KVStoreClosure {
+
+    private final KVStoreClosure done;
+    private final String         regionId;
+    private final byte           kvOp;
+    private final long           keysCount;
+    private final long           bytesWritten;
+    private final Timer.Context  ctx;
+    private final Timer.Context  opCtx;
+
+    public MetricsKVClosureAdapter(KVStoreClosure done, String regionId, byte kvOp, long keysCount, long bytesWritten,
+                                   Timer.Context ctx) {
+        this.done = done;
+        this.regionId = regionId;
+        this.kvOp = kvOp;
+        this.keysCount = keysCount;
+        this.bytesWritten = bytesWritten;
+        this.ctx = ctx;
+        this.opCtx = opTimeCtx(kvOp);
+    }
+
+    @Override
+    public Errors getError() {
+        if (this.done != null) {
+            return this.done.getError();
+        }
+        return null;
+    }
+
+    @Override
+    public void setError(Errors error) {
+        if (this.done != null) {
+            this.done.setError(error);
+        }
+    }
+
+    @Override
+    public Object getData() {
+        if (this.done != null) {
+            return this.done.getData();
+        }
+        return null;
+    }
+
+    @Override
+    public void setData(Object data) {
+        if (this.done != null) {
+            this.done.setData(data);
+        }
+    }
+
+    @Override
+    public void run(final Status status) {
+        try {
+            if (this.done != null) {
+                this.done.run(status);
+            }
+        } finally {
+            if (status.isOk()) {
+                doStatistics();
+            }
+            this.ctx.stop();
+            this.opCtx.stop();
+        }
+    }
+
+    private Timer.Context opTimeCtx(final byte op) {
+        return KVMetrics.timer(RPC_REQUEST_HANDLE_TIMER, this.regionId, KVOperation.opName(op)).time();
+    }
+
+    @SuppressWarnings("unchecked")
+    private void doStatistics() {
+        final String id = this.regionId; // stack copy
+        switch (this.kvOp) {
+            case KVOperation.PUT:
+            case KVOperation.MERGE: {
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc();
+                KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                break;
+            }
+            case KVOperation.PUT_IF_ABSENT: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                byte[] bytes = (byte[]) getData();
+                if (bytes != null) {
+                    KVMetrics.counter(REGION_BYTES_READ, id).inc(bytes.length);
+                } else {
+                    KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc();
+                    KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                }
+                break;
+            }
+            case KVOperation.DELETE:
+            case KVOperation.RESET_SEQUENCE:
+            case KVOperation.KEY_LOCK_RELEASE: {
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc();
+                break;
+            }
+            case KVOperation.PUT_LIST: {
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc(keysCount);
+                KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                break;
+            }
+            case KVOperation.DELETE_RANGE: {
+                // TODO if this is needed?
+                break;
+            }
+            case KVOperation.DELETE_LIST: {
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc(keysCount);
+                break;
+            }
+            case KVOperation.GET_SEQUENCE:
+            case KVOperation.KEY_LOCK: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc(); // maybe 0
+                KVMetrics.counter(REGION_BYTES_READ, id).inc(this.bytesWritten);
+                KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                break;
+            }
+            case KVOperation.GET: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                final byte[] data = (byte[]) getData();
+                if (data != null) {
+                    KVMetrics.counter(REGION_BYTES_READ, id).inc(data.length);
+                }
+                break;
+            }
+            case KVOperation.MULTI_GET: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc(this.keysCount);
+                final Map data = (Map) getData();
+                if (data != null) {
+                    long bytesRead = 0;
+                    for (final byte[] bytes : data.values()) {
+                        if (bytes == null) {
+                            continue;
+                        }
+                        bytesRead += bytes.length;
+                    }
+                    KVMetrics.counter(REGION_BYTES_READ, id).inc(bytesRead);
+                }
+                break;
+            }
+            case KVOperation.CONTAINS_KEY: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                break;
+            }
+            case KVOperation.SCAN: {
+                final List data = (List) getData();
+                if (data != null) {
+                    long bytesRead = 0;
+                    for (final KVEntry kvEntry : data) {
+                        final byte[] value = kvEntry.getValue();
+                        if (value == null) {
+                            continue;
+                        }
+                        bytesRead += value.length;
+                    }
+                    KVMetrics.counter(REGION_KEYS_READ, id).inc(data.size());
+                    KVMetrics.counter(REGION_BYTES_READ, id).inc(bytesRead);
+                }
+                break;
+            }
+            case KVOperation.GET_PUT: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc();
+                final byte[] data = (byte[]) getData();
+                if (data != null) {
+                    KVMetrics.counter(REGION_BYTES_READ, id).inc(data.length);
+                }
+                KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                break;
+            }
+            case KVOperation.COMPARE_PUT: {
+                KVMetrics.counter(REGION_KEYS_READ, id).inc();
+                final Boolean data = (Boolean) getData();
+                if (data != null && data) {
+                    KVMetrics.counter(REGION_KEYS_WRITTEN, id).inc();
+                    KVMetrics.counter(REGION_BYTES_WRITTEN, id).inc(this.bytesWritten);
+                }
+                break;
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsRawKVStore.java
new file mode 100644
index 0000000..0ead193
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/MetricsRawKVStore.java
@@ -0,0 +1,277 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.metrics.KVMetrics;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.codahale.metrics.Timer;
+
+import static com.alipay.sofa.jraft.rhea.metrics.KVMetricNames.RPC_REQUEST_HANDLE_TIMER;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.COMPARE_PUT;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.COMPARE_PUT_ALL;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.CONTAINS_KEY;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.DELETE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.DELETE_LIST;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.DELETE_RANGE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.GET;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.GET_PUT;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.GET_SEQUENCE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.KEY_LOCK;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.KEY_LOCK_RELEASE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.MERGE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.MULTI_GET;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.NODE_EXECUTE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.PUT;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.PUT_IF_ABSENT;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.PUT_LIST;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.RESET_SEQUENCE;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.REVERSE_SCAN;
+import static com.alipay.sofa.jraft.rhea.storage.KVOperation.SCAN;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MetricsRawKVStore implements RawKVStore {
+
+    private final String     regionId;
+    private final RawKVStore rawKVStore;
+    private final Timer      timer;
+
+    public MetricsRawKVStore(long regionId, RawKVStore rawKVStore) {
+        this.regionId = String.valueOf(regionId);
+        this.rawKVStore = rawKVStore;
+        this.timer = KVMetrics.timer(RPC_REQUEST_HANDLE_TIMER, this.regionId);
+    }
+
+    @Override
+    public KVIterator localIterator() {
+        return this.rawKVStore.localIterator();
+    }
+
+    @Override
+    public void get(final byte[] key, final KVStoreClosure closure) {
+        get(key, true, closure);
+    }
+
+    @Override
+    public void get(final byte[] key, final boolean readOnlySafe, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, GET, 1, 0);
+        this.rawKVStore.get(key, readOnlySafe, c);
+    }
+
+    @Override
+    public void multiGet(final List keys, final KVStoreClosure closure) {
+        multiGet(keys, true, closure);
+    }
+
+    @Override
+    public void multiGet(final List keys, final boolean readOnlySafe, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, MULTI_GET, keys.size(), 0);
+        this.rawKVStore.multiGet(keys, readOnlySafe, c);
+    }
+
+    @Override
+    public void containsKey(final byte[] key, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, CONTAINS_KEY, 1, 0);
+        this.rawKVStore.containsKey(key, c);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final boolean returnValue, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, SCAN, 0, 0);
+        this.rawKVStore.scan(startKey, endKey, limit, readOnlySafe, returnValue, c);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final boolean returnValue, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                            final boolean returnValue, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, REVERSE_SCAN, 0, 0);
+        this.rawKVStore.reverseScan(startKey, endKey, limit, readOnlySafe, returnValue, c);
+    }
+
+    @Override
+    public void getSequence(final byte[] seqKey, final int step, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, GET_SEQUENCE, 1, 8);
+        this.rawKVStore.getSequence(seqKey, step, c);
+    }
+
+    @Override
+    public void resetSequence(final byte[] seqKey, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, RESET_SEQUENCE, 1, 0);
+        this.rawKVStore.resetSequence(seqKey, c);
+    }
+
+    @Override
+    public void put(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, PUT, 1, value.length);
+        this.rawKVStore.put(key, value, c);
+    }
+
+    @Override
+    public void getAndPut(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, GET_PUT, 1, value.length);
+        this.rawKVStore.getAndPut(key, value, c);
+    }
+
+    @Override
+    public void compareAndPut(final byte[] key, final byte[] expect, final byte[] update, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, COMPARE_PUT, 1, update.length);
+        this.rawKVStore.compareAndPut(key, expect, update, c);
+    }
+
+    @Override
+    public void merge(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, MERGE, 1, value.length);
+        this.rawKVStore.merge(key, value, c);
+    }
+
+    @Override
+    public void put(final List entries, final KVStoreClosure closure) {
+        long bytesWritten = 0;
+        for (final KVEntry kvEntry : entries) {
+            byte[] value = kvEntry.getValue();
+            bytesWritten += (value == null ? 0 : value.length);
+        }
+        final KVStoreClosure c = metricsAdapter(closure, PUT_LIST, entries.size(), bytesWritten);
+        this.rawKVStore.put(entries, c);
+    }
+
+    @Override
+    public void compareAndPutAll(final List entries, final KVStoreClosure closure) {
+        long bytesWritten = 0;
+        for (final CASEntry casEntry : entries) {
+            byte[] value = casEntry.getUpdate();
+            bytesWritten += (value == null ? 0 : value.length);
+        }
+        final KVStoreClosure c = metricsAdapter(closure, COMPARE_PUT_ALL, entries.size(), bytesWritten);
+        this.rawKVStore.compareAndPutAll(entries, c);
+    }
+
+    @Override
+    public void putIfAbsent(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, PUT_IF_ABSENT, 1, value.length);
+        this.rawKVStore.putIfAbsent(key, value, c);
+    }
+
+    @Override
+    public void tryLockWith(final byte[] key, final byte[] fencingKey, final boolean keepLease,
+                            final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        // 'keysCount' and 'bytesWritten' can't be provided with exact numbers, but I endured
+        final KVStoreClosure c = metricsAdapter(closure, KEY_LOCK, 2, 0);
+        this.rawKVStore.tryLockWith(key, fencingKey, keepLease, acquirer, c);
+    }
+
+    @Override
+    public void releaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        // 'keysCount' and 'bytesWritten' can't be provided with exact numbers, but I endured
+        final KVStoreClosure c = metricsAdapter(closure, KEY_LOCK_RELEASE, 2, 0);
+        this.rawKVStore.releaseLockWith(key, acquirer, c);
+    }
+
+    @Override
+    public void delete(final byte[] key, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, DELETE, 1, 0);
+        this.rawKVStore.delete(key, c);
+    }
+
+    @Override
+    public void deleteRange(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, DELETE_RANGE, 0, 0);
+        this.rawKVStore.deleteRange(startKey, endKey, c);
+    }
+
+    @Override
+    public void delete(final List keys, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, DELETE_LIST, keys.size(), 0);
+        this.rawKVStore.delete(keys, c);
+    }
+
+    @Override
+    public void execute(final NodeExecutor nodeExecutor, final boolean isLeader, final KVStoreClosure closure) {
+        final KVStoreClosure c = metricsAdapter(closure, NODE_EXECUTE, 0, 0);
+        this.rawKVStore.execute(nodeExecutor, isLeader, c);
+    }
+
+    private MetricsKVClosureAdapter metricsAdapter(final KVStoreClosure closure, final byte op, final long keysCount,
+                                                   final long bytesWritten) {
+        return new MetricsKVClosureAdapter(closure, this.regionId, op, keysCount, bytesWritten, timeCtx());
+    }
+
+    private Timer.Context timeCtx() {
+        return this.timer.time();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/NodeExecutor.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/NodeExecutor.java
new file mode 100644
index 0000000..ad3c3d7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/NodeExecutor.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+
+import com.alipay.sofa.jraft.Status;
+
+/**
+ * This is a callback interface, {@link NodeExecutor#execute(Status, boolean)}
+ * will be triggered when each node's state machine is applied.
+ *
+ * Note that any element contained in the implementation of this interface must
+ * implement the {@link Serializable} interface.
+ *
+ * @author jiachun.fjc
+ */
+public interface NodeExecutor extends Serializable {
+
+    /**
+     * This callback method will be triggered when each node's state machine
+     * is applied.
+     *
+     * @param status   The execution state of the current node
+     * @param isLeader Whether the current node is a leader
+     */
+    void execute(Status status, boolean isLeader);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RaftRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RaftRawKVStore.java
new file mode 100644
index 0000000..2d27141
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RaftRawKVStore.java
@@ -0,0 +1,392 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.util.Clock;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ * KVStore based on RAFT replica state machine.
+ *
+ * @author jiachun.fjc
+ */
+public class RaftRawKVStore implements RawKVStore {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RaftRawKVStore.class);
+
+    private final Node          node;
+    private final RawKVStore    kvStore;
+    private final Executor      readIndexExecutor;
+
+    public RaftRawKVStore(Node node, RawKVStore kvStore, Executor readIndexExecutor) {
+        this.node = node;
+        this.kvStore = kvStore;
+        this.readIndexExecutor = readIndexExecutor;
+    }
+
+    @Override
+    public KVIterator localIterator() {
+        return this.kvStore.localIterator();
+    }
+
+    @Override
+    public void get(final byte[] key, final KVStoreClosure closure) {
+        get(key, true, closure);
+    }
+
+    @Override
+    public void get(final byte[] key, final boolean readOnlySafe, final KVStoreClosure closure) {
+        if (!readOnlySafe) {
+            this.kvStore.get(key, false, closure);
+            return;
+        }
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.get(key, true, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [get] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createGet(key), closure);
+                    } else {
+                        LOG.warn("Fail to [get] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void multiGet(final List keys, final KVStoreClosure closure) {
+        multiGet(keys, true, closure);
+    }
+
+    @Override
+    public void multiGet(final List keys, final boolean readOnlySafe, final KVStoreClosure closure) {
+        if (!readOnlySafe) {
+            this.kvStore.multiGet(keys, false, closure);
+            return;
+        }
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.multiGet(keys, true, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [multiGet] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createMultiGet(keys), closure);
+                    } else {
+                        LOG.warn("Fail to [multiGet] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void containsKey(final byte[] key, final KVStoreClosure closure) {
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.containsKey(key, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [containsKey] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createContainsKey(key), closure);
+                    } else {
+                        LOG.warn("Fail to [containsKey] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final KVStoreClosure closure) {
+        scan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final boolean returnValue, final KVStoreClosure closure) {
+        if (!readOnlySafe) {
+            this.kvStore.scan(startKey, endKey, limit, false, returnValue, closure);
+            return;
+        }
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.scan(startKey, endKey, limit, true, returnValue, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [scan] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createScan(startKey, endKey, limit, returnValue), closure);
+                    } else {
+                        LOG.warn("Fail to [scan] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                            final boolean returnValue, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                            final KVStoreClosure closure) {
+        reverseScan(startKey, endKey, limit, readOnlySafe, true, closure);
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                            final boolean returnValue, final KVStoreClosure closure) {
+        if (!readOnlySafe) {
+            this.kvStore.reverseScan(startKey, endKey, limit, false, returnValue, closure);
+            return;
+        }
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.reverseScan(startKey, endKey, limit, true, returnValue, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [reverseScan] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createReverseScan(startKey, endKey, limit, returnValue), closure);
+                    } else {
+                        LOG.warn("Fail to [reverseScan] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void getSequence(final byte[] seqKey, final int step, final KVStoreClosure closure) {
+        if (step > 0) {
+            applyOperation(KVOperation.createGetSequence(seqKey, step), closure);
+            return;
+        }
+        // read-only (step==0)
+        this.node.readIndex(BytesUtil.EMPTY_BYTES, new ReadIndexClosure() {
+
+            @Override
+            public void run(final Status status, final long index, final byte[] reqCtx) {
+                if (status.isOk()) {
+                    RaftRawKVStore.this.kvStore.getSequence(seqKey, 0, closure);
+                    return;
+                }
+                RaftRawKVStore.this.readIndexExecutor.execute(() -> {
+                    if (isLeader()) {
+                        LOG.warn("Fail to [getSequence] with 'ReadIndex': {}, try to applying to the state machine.", status);
+                        // If 'read index' read fails, try to applying to the state machine at the leader node
+                        applyOperation(KVOperation.createGetSequence(seqKey, 0), closure);
+                    } else {
+                        LOG.warn("Fail to [getSequence] with 'ReadIndex': {}.", status);
+                        // Client will retry to leader node
+                        new KVClosureAdapter(closure, null).run(status);
+                    }
+                });
+            }
+        });
+    }
+
+    @Override
+    public void resetSequence(final byte[] seqKey, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createResetSequence(seqKey), closure);
+    }
+
+    @Override
+    public void put(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createPut(key, value), closure);
+    }
+
+    @Override
+    public void getAndPut(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createGetAndPut(key, value), closure);
+    }
+
+    @Override
+    public void compareAndPut(final byte[] key, final byte[] expect, final byte[] update, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createCompareAndPut(key, expect, update), closure);
+    }
+
+    @Override
+    public void merge(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createMerge(key, value), closure);
+    }
+
+    @Override
+    public void put(final List entries, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createPutList(entries), closure);
+    }
+
+    @Override
+    public void compareAndPutAll(final List entries, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createCompareAndPutAll(entries), closure);
+    }
+
+    @Override
+    public void putIfAbsent(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createPutIfAbsent(key, value), closure);
+    }
+
+    @Override
+    public void tryLockWith(final byte[] key, final byte[] fencingKey, final boolean keepLease,
+                            final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        // The algorithm relies on the assumption that while there is no
+        // synchronized clock across the processes, still the local time in
+        // every process flows approximately at the same rate, with an error
+        // which is small compared to the auto-release time of the lock.
+        acquirer.setLockingTimestamp(Clock.defaultClock().getTime());
+        applyOperation(KVOperation.createKeyLockRequest(key, fencingKey, Pair.of(keepLease, acquirer)), closure);
+    }
+
+    @Override
+    public void releaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createKeyLockReleaseRequest(key, acquirer), closure);
+    }
+
+    @Override
+    public void delete(final byte[] key, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createDelete(key), closure);
+    }
+
+    @Override
+    public void deleteRange(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createDeleteRange(startKey, endKey), closure);
+    }
+
+    @Override
+    public void delete(final List keys, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createDeleteList(keys), closure);
+    }
+
+    @Override
+    public void execute(final NodeExecutor nodeExecutor, final boolean isLeader, final KVStoreClosure closure) {
+        applyOperation(KVOperation.createNodeExecutor(nodeExecutor), closure);
+    }
+
+    private void applyOperation(final KVOperation op, final KVStoreClosure closure) {
+        if (!isLeader()) {
+            closure.setError(Errors.NOT_LEADER);
+            closure.run(new Status(RaftError.EPERM, "Not leader"));
+            return;
+        }
+        final Task task = new Task();
+        task.setData(ByteBuffer.wrap(Serializers.getDefault().writeObject(op)));
+        task.setDone(new KVClosureAdapter(closure, op));
+        this.node.apply(task);
+    }
+
+    private boolean isLeader() {
+        return this.node.isLeader(false);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RawKVStore.java
new file mode 100644
index 0000000..5454fbf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RawKVStore.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * Raw KV store
+ *
+ * @author dennis
+ * @author jiachun.fjc
+ */
+public interface RawKVStore {
+
+    /**
+     * Returns a heap-allocated iterator over the contents of the
+     * database.
+     *
+     * Caller should close the iterator when it is no longer needed.
+     * The returned iterator should be closed before this db is closed.
+     *
+     * 
+     *     KVIterator it = unsafeLocalIterator();
+     *     try {
+     *         // do something
+     *     } finally {
+     *         it.close();
+     *     }
+     * 
+     */
+    KVIterator localIterator();
+
+    /**
+     * Equivalent to {@code get(key, true, closure)}.
+     */
+    void get(final byte[] key, final KVStoreClosure closure);
+
+    /**
+     * Get which returns a new byte array storing the value associated
+     * with the specified input key if any.  null will be returned if
+     * the specified key is not found.
+     *
+     * Provide consistent reading if {@code readOnlySafe} is true.
+     */
+    void get(final byte[] key, final boolean readOnlySafe, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code multiGet(keys, true, closure)}.
+     */
+    void multiGet(final List keys, final KVStoreClosure closure);
+
+    /**
+     * Returns a map of keys for which values were found in DB.
+     *
+     * The performance is similar to get(), multiGet() reads from the
+     * same consistent view, but it is not faster.
+     *
+     * Provide consistent reading if {@code readOnlySafe} is true.
+     */
+    void multiGet(final List keys, final boolean readOnlySafe, final KVStoreClosure closure);
+
+    /**
+     * Returns whether DB contains the specified input key {@code key}.
+     */
+    void containsKey(final byte[] key, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, Integer.MAX_VALUE, closure)}.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure)}.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure)}.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final boolean returnValue,
+              final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, limit, true, closure)}.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code scan(startKey, endKey, limit, readOnlySafe, true, closure)}.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+              final KVStoreClosure closure);
+
+    /**
+     * Query all data in the key range of [startKey, endKey),
+     * {@code limit} is the max number of keys.
+     *
+     * Provide consistent reading if {@code readOnlySafe} is true.
+     *
+     * Only return keys(ignore values) if {@code returnValue} is false.
+     */
+    void scan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+              final boolean returnValue, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, Integer.MAX_VALUE, closure)}.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, closure)}.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe,
+                     final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, Integer.MAX_VALUE, readOnlySafe, returnValue, closure)}.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, limit, true, closure)}.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final KVStoreClosure closure);
+
+    /**
+     * Equivalent to {@code reverseScan(startKey, endKey, limit, readOnlySafe, true, closure)}.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final KVStoreClosure closure);
+
+    /**
+     * Reverse query all data in the key range of [startKey, endKey),
+     * {@code limit} is the max number of keys.
+     *
+     * Provide consistent reading if {@code readOnlySafe} is true.
+     *
+     * Only return keys(ignore values) if {@code returnValue} is false.
+     */
+    void reverseScan(final byte[] startKey, final byte[] endKey, final int limit, final boolean readOnlySafe,
+                     final boolean returnValue, final KVStoreClosure closure);
+
+    /**
+     * Get a globally unique auto-increment sequence.
+     *
+     * Be careful do not to try to get or update the value of {@code seqKey}
+     * by other methods, you won't get it.
+     */
+    void getSequence(final byte[] seqKey, final int step, final KVStoreClosure closure);
+
+    /**
+     * Reset the sequence to 0.
+     */
+    void resetSequence(final byte[] seqKey, final KVStoreClosure closure);
+
+    /**
+     * Set the database entry for "key" to "value".
+     */
+    void put(final byte[] key, final byte[] value, final KVStoreClosure closure);
+
+    /**
+     * Set the database entry for "key" to "value", and return the
+     * previous value associated with "key", or null if there was no
+     * mapping for "key".
+     */
+    void getAndPut(final byte[] key, final byte[] value, final KVStoreClosure closure);
+
+    /**
+     * Atomically sets the value to the given updated value
+     * if the current value equal (compare bytes) the expected value.
+     */
+    void compareAndPut(final byte[] key, final byte[] expect, final byte[] update, final KVStoreClosure closure);
+
+    /**
+     * Add merge operand for key/value pair.
+     *
+     *  
+     *     // Writing aa under key
+     *     db.put("key", "aa");
+     *
+     *     // Writing bb under key
+     *     db.merge("key", "bb");
+     *
+     *     assertThat(db.get("key")).isEqualTo("aa,bb");
+     * 

+     */
+    void merge(final byte[] key, final byte[] value, final KVStoreClosure closure);
+
+    /**
+     * Store key/value pairs in batch.
+     */
+    void put(final List entries, final KVStoreClosure closure);
+
+    /**
+     * CAS in batch.
+     */
+    void compareAndPutAll(final List entries, final KVStoreClosure closure);
+
+    /**
+     * If the specified key is not already associated with a value
+     * associates it with the given value and returns (closure.data)
+     * {@code null}, else returns the current value.
+     */
+    void putIfAbsent(final byte[] key, final byte[] value, final KVStoreClosure closure);
+
+    /**
+     * Tries to lock the specified key, must contain a timeout
+     */
+    void tryLockWith(final byte[] key, final byte[] fencingKey, final boolean keepLease,
+                     final DistributedLock.Acquirer acquirer, final KVStoreClosure closure);
+
+    /**
+     * Unlock the specified key with lock.
+     */
+    void releaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer, final KVStoreClosure closure);
+
+    /**
+     * Delete data by the {@code key}.
+     */
+    void delete(final byte[] key, final KVStoreClosure closure);
+
+    /**
+     * Delete all data in the key range of [startKey, endKey).
+     */
+    void deleteRange(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure);
+
+    /**
+     * Delete data by the {@code keys} in batch.
+     */
+    void delete(final List keys, final KVStoreClosure closure);
+
+    /**
+     * The {@code nodeExecutor} will be triggered when each node's
+     * state machine is applied.
+     */
+    void execute(final NodeExecutor nodeExecutor, final boolean isLeader, final KVStoreClosure closure);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksDBBackupInfo.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksDBBackupInfo.java
new file mode 100644
index 0000000..163eaf4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksDBBackupInfo.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+
+import org.rocksdb.BackupInfo;
+
+/**
+ * RocksDB backup metadata info.
+ *
+ * @author dennis
+ * @author jiachun.fjc
+ */
+public class RocksDBBackupInfo implements Serializable {
+
+    private static final long serialVersionUID = -7010741841443565098L;
+
+    private int               backupId;
+    private int               numberFiles;
+    private long              timestamp;
+    private long              size;
+
+    public RocksDBBackupInfo(BackupInfo info) {
+        super();
+        this.size = info.size();
+        this.backupId = info.backupId();
+        this.timestamp = info.timestamp();
+        this.numberFiles = info.numberFiles();
+    }
+
+    public int getBackupId() {
+        return backupId;
+    }
+
+    public void setBackupId(int backupId) {
+        this.backupId = backupId;
+    }
+
+    public RocksDBBackupInfo() {
+        super();
+    }
+
+    public int getNumberFiles() {
+        return numberFiles;
+    }
+
+    public void setNumberFiles(int numberFiles) {
+        this.numberFiles = numberFiles;
+    }
+
+    public long getTimestamp() {
+        return timestamp;
+    }
+
+    public void setTimestamp(long timestamp) {
+        this.timestamp = timestamp;
+    }
+
+    public long getSize() {
+        return size;
+    }
+
+    public void setSize(long size) {
+        this.size = size;
+    }
+
+    @Override
+    public String toString() {
+        return "RocksDBBackupInfo{" + "backupId=" + backupId + ", numberFiles=" + numberFiles + ", timestamp="
+               + timestamp + ", size=" + size + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVIterator.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVIterator.java
new file mode 100644
index 0000000..68ba9ac
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVIterator.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.concurrent.locks.Lock;
+
+import org.rocksdb.RocksIterator;
+
+import com.alipay.sofa.jraft.rhea.errors.InvalidIteratorVersion;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RocksKVIterator implements KVIterator {
+
+    private final RocksRawKVStore rocksRawKVStore;
+    private final RocksIterator   it;
+    private final Lock            dbReadLock;
+    private final long            dbVersion;
+
+    public RocksKVIterator(RocksRawKVStore rocksRawKVStore, RocksIterator it, Lock dbReadLock, long dbVersion) {
+        this.rocksRawKVStore = rocksRawKVStore;
+        this.it = it;
+        this.dbReadLock = dbReadLock;
+        this.dbVersion = dbVersion;
+    }
+
+    @Override
+    public boolean isValid() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            return it.isValid();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void seekToFirst() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.seekToFirst();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void seekToLast() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.seekToLast();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void seek(final byte[] target) {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.seek(target);
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void seekForPrev(final byte[] target) {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.seekForPrev(target);
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void next() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.next();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void prev() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.prev();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public byte[] key() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            return it.key();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public byte[] value() {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            return it.value();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void close() throws Exception {
+        final Lock readLock = this.dbReadLock;
+        readLock.lock();
+        try {
+            ensureSafety();
+            it.close();
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    private void ensureSafety() {
+        if (this.dbVersion != this.rocksRawKVStore.getDatabaseVersion()) {
+            throw new InvalidIteratorVersion("current iterator is belong to the older version of db: " + this.dbVersion
+                                             + ", the newest db version: " + this.rocksRawKVStore.getDatabaseVersion());
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVStoreSnapshotFile.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVStoreSnapshotFile.java
new file mode 100644
index 0000000..980ae2a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksKVStoreSnapshotFile.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+import com.alipay.sofa.jraft.rhea.errors.StorageException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.util.RegionHelper;
+
+import static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RocksKVStoreSnapshotFile extends AbstractKVStoreSnapshotFile {
+
+    private final RocksRawKVStore kvStore;
+
+    RocksKVStoreSnapshotFile(RocksRawKVStore kvStore) {
+        this.kvStore = kvStore;
+    }
+
+    @Override
+    CompletableFuture doSnapshotSave(final String snapshotPath, final Region region,
+                                                            final ExecutorService executor) throws Exception {
+        if (RegionHelper.isMultiGroup(region)) {
+            final CompletableFuture snapshotFuture = this.kvStore.writeSstSnapshot(snapshotPath, region, executor);
+            final CompletableFuture metaFuture = new CompletableFuture<>();
+            snapshotFuture.whenComplete((aVoid, throwable) -> {
+                if (throwable == null) {
+                    metaFuture.complete(writeMetadata(region));
+                } else {
+                    metaFuture.completeExceptionally(throwable);
+                }
+            });
+            return metaFuture;
+        }
+        if (this.kvStore.isFastSnapshot()) {
+            // Checkpoint is fast enough, no need to asynchronous
+            this.kvStore.writeSnapshot(snapshotPath);
+            return CompletableFuture.completedFuture(writeMetadata(null));
+        }
+        final RocksDBBackupInfo backupInfo = this.kvStore.backupDB(snapshotPath);
+        return CompletableFuture.completedFuture(writeMetadata(backupInfo));
+    }
+
+    @Override
+    void doSnapshotLoad(final String snapshotPath, final LocalFileMeta meta, final Region region) throws Exception {
+        if (RegionHelper.isMultiGroup(region)) {
+            final Region snapshotRegion = readMetadata(meta, Region.class);
+            if (!RegionHelper.isSameRange(region, snapshotRegion)) {
+                throw new StorageException("Invalid snapshot region: " + snapshotRegion + " current region is: "
+                                           + region);
+            }
+            this.kvStore.readSstSnapshot(snapshotPath);
+            return;
+        }
+        if (this.kvStore.isFastSnapshot()) {
+            this.kvStore.readSnapshot(snapshotPath);
+            return;
+        }
+        final RocksDBBackupInfo rocksBackupInfo = readMetadata(meta, RocksDBBackupInfo.class);
+        this.kvStore.restoreBackup(snapshotPath, rocksBackupInfo);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksRawKVStore.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksRawKVStore.java
new file mode 100644
index 0000000..aea0276
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/RocksRawKVStore.java
@@ -0,0 +1,1667 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.Function;
+
+import org.apache.commons.io.FileUtils;
+import org.rocksdb.BackupEngine;
+import org.rocksdb.BackupInfo;
+import org.rocksdb.BackupableDBOptions;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.Checkpoint;
+import org.rocksdb.ColumnFamilyDescriptor;
+import org.rocksdb.ColumnFamilyHandle;
+import org.rocksdb.ColumnFamilyOptions;
+import org.rocksdb.DBOptions;
+import org.rocksdb.Env;
+import org.rocksdb.EnvOptions;
+import org.rocksdb.Holder;
+import org.rocksdb.IngestExternalFileOptions;
+import org.rocksdb.Options;
+import org.rocksdb.ReadOptions;
+import org.rocksdb.RestoreOptions;
+import org.rocksdb.RocksDB;
+import org.rocksdb.RocksDBException;
+import org.rocksdb.RocksIterator;
+import org.rocksdb.Snapshot;
+import org.rocksdb.SstFileWriter;
+import org.rocksdb.Statistics;
+import org.rocksdb.StatisticsCollectorCallback;
+import org.rocksdb.StatsCollectorInput;
+import org.rocksdb.StringAppendOperator;
+import org.rocksdb.WriteBatch;
+import org.rocksdb.WriteOptions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.errors.StorageException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.rocks.support.RocksStatisticsCollector;
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.Partitions;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.DebugStatistics;
+import com.alipay.sofa.jraft.util.Describer;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.StorageOptionsFactory;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.alipay.sofa.jraft.util.Utils;
+import com.alipay.sofa.jraft.util.concurrent.AdjustableSemaphore;
+import com.codahale.metrics.Timer;
+
+/**
+ * Local KV store based on RocksDB
+ *
+ * @author dennis
+ * @author jiachun.fjc
+ */
+public class RocksRawKVStore extends BatchRawKVStore implements Describer {
+
+    private static final Logger                LOG                  = LoggerFactory.getLogger(RocksRawKVStore.class);
+
+    static {
+        RocksDB.loadLibrary();
+    }
+
+    // The maximum number of keys in once batch write
+    public static final int                    MAX_BATCH_WRITE_SIZE = SystemPropertyUtil.getInt(
+                                                                        "rhea.rocksdb.user.max_batch_write_size", 128);
+
+    private final AdjustableSemaphore          shutdownLock         = new AdjustableSemaphore();
+    private final ReadWriteLock                readWriteLock        = new ReentrantReadWriteLock();
+
+    private final AtomicLong                   databaseVersion      = new AtomicLong(0);
+    private final Serializer                   serializer           = Serializers.getDefault();
+
+    private final List    cfOptionsList        = Lists.newArrayList();
+    private final List cfDescriptors        = Lists.newArrayList();
+
+    private ColumnFamilyHandle                 defaultHandle;
+    private ColumnFamilyHandle                 sequenceHandle;
+    private ColumnFamilyHandle                 lockingHandle;
+    private ColumnFamilyHandle                 fencingHandle;
+
+    private RocksDB                            db;
+
+    private RocksDBOptions                     opts;
+    private DBOptions                          options;
+    private WriteOptions                       writeOptions;
+    private DebugStatistics                    statistics;
+    private RocksStatisticsCollector           statisticsCollector;
+
+    @Override
+    public boolean init(final RocksDBOptions opts) {
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try {
+            if (this.db != null) {
+                LOG.info("[RocksRawKVStore] already started.");
+                return true;
+            }
+            this.opts = opts;
+            this.options = createDBOptions();
+            if (opts.isOpenStatisticsCollector()) {
+                this.statistics = new DebugStatistics();
+                this.options.setStatistics(this.statistics);
+                final long intervalSeconds = opts.getStatisticsCallbackIntervalSeconds();
+                if (intervalSeconds > 0) {
+                    this.statisticsCollector = new RocksStatisticsCollector(TimeUnit.SECONDS.toMillis(intervalSeconds));
+                    this.statisticsCollector.start();
+                }
+            }
+            final ColumnFamilyOptions cfOptions = createColumnFamilyOptions();
+            this.cfOptionsList.add(cfOptions);
+            // default column family
+            this.cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cfOptions));
+            // sequence column family
+            this.cfDescriptors.add(new ColumnFamilyDescriptor(BytesUtil.writeUtf8("RHEA_SEQUENCE"), cfOptions));
+            // locking column family
+            this.cfDescriptors.add(new ColumnFamilyDescriptor(BytesUtil.writeUtf8("RHEA_LOCKING"), cfOptions));
+            // fencing column family
+            this.cfDescriptors.add(new ColumnFamilyDescriptor(BytesUtil.writeUtf8("RHEA_FENCING"), cfOptions));
+            this.writeOptions = new WriteOptions();
+            this.writeOptions.setSync(opts.isSync());
+            // If `sync` is true, `disableWAL` must be set false.
+            this.writeOptions.setDisableWAL(!opts.isSync() && opts.isDisableWAL());
+            // Delete existing data, relying on raft's snapshot and log playback
+            // to reply to the data is the correct behavior.
+            destroyRocksDB(opts);
+            openRocksDB(opts);
+            this.shutdownLock.setMaxPermits(1);
+            LOG.info("[RocksRawKVStore] start successfully, options: {}.", opts);
+            return true;
+        } catch (final Exception e) {
+            LOG.error("Fail to open rocksDB at path {}, {}.", opts.getDbPath(), StackTraceUtil.stackTrace(e));
+        } finally {
+            writeLock.unlock();
+        }
+        return false;
+    }
+
+    @Override
+    public void shutdown() {
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try {
+            if (this.db == null) {
+                return;
+            }
+            this.shutdownLock.setMaxPermits(0);
+            closeRocksDB();
+            if (this.defaultHandle != null) {
+                this.defaultHandle.close();
+                this.defaultHandle = null;
+            }
+            if (this.sequenceHandle != null) {
+                this.sequenceHandle.close();
+                this.sequenceHandle = null;
+            }
+            if (this.lockingHandle != null) {
+                this.lockingHandle.close();
+                this.lockingHandle = null;
+            }
+            if (this.fencingHandle != null) {
+                this.fencingHandle.close();
+                this.fencingHandle = null;
+            }
+            for (final ColumnFamilyOptions cfOptions : this.cfOptionsList) {
+                cfOptions.close();
+            }
+            this.cfOptionsList.clear();
+            this.cfDescriptors.clear();
+            if (this.options != null) {
+                this.options.close();
+                this.options = null;
+            }
+            if (this.statisticsCollector != null) {
+                try {
+                    this.statisticsCollector.shutdown(3000);
+                } catch (final Throwable ignored) {
+                    // ignored
+                }
+            }
+            if (this.statistics != null) {
+                this.statistics.close();
+                this.statistics = null;
+            }
+            if (this.writeOptions != null) {
+                this.writeOptions.close();
+                this.writeOptions = null;
+            }
+        } finally {
+            writeLock.unlock();
+            LOG.info("[RocksRawKVStore] shutdown successfully.");
+        }
+    }
+
+    @Override
+    public KVIterator localIterator() {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            return new RocksKVIterator(this, this.db.newIterator(), readLock, getDatabaseVersion());
+        } finally {
+            readLock.unlock();
+        }
+    }
+
+    @Override
+    public void get(final byte[] key, @SuppressWarnings("unused") final boolean readOnlySafe,
+                    final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] value = this.db.get(key);
+            setSuccess(closure, value);
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET], key: [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [GET]");
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void multiGet(final List keys, @SuppressWarnings("unused") final boolean readOnlySafe,
+                         final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("MULTI_GET");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final Map rawMap = this.db.multiGet(keys);
+            final Map resultMap = Maps.newHashMapWithExpectedSize(rawMap.size());
+            for (final Map.Entry entry : rawMap.entrySet()) {
+                resultMap.put(ByteArray.wrap(entry.getKey()), entry.getValue());
+            }
+            setSuccess(closure, resultMap);
+        } catch (final Exception e) {
+            LOG.error("Fail to [MULTI_GET], key size: [{}], {}.", keys.size(), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [MULTI_GET]");
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void containsKey(final byte[] key, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("CONTAINS_KEY");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            boolean exists = false;
+            Holder valueHolder = new Holder<>();
+            if (this.db.keyMayExist(key, valueHolder)) {
+                exists = valueHolder.getValue() != null;
+            }
+            setSuccess(closure, exists);
+        } catch (final Exception e) {
+            LOG.error("Fail to [CONTAINS_KEY], key: [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [CONTAINS_KEY]");
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void scan(final byte[] startKey, final byte[] endKey, final int limit,
+                     @SuppressWarnings("unused") final boolean readOnlySafe, final boolean returnValue,
+                     final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("SCAN");
+        final List entries = Lists.newArrayList();
+        final int maxCount = normalizeLimit(limit);
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (final RocksIterator it = this.db.newIterator()) {
+            if (startKey == null) {
+                it.seekToFirst();
+            } else {
+                it.seek(startKey);
+            }
+            int count = 0;
+            while (it.isValid() && count++ < maxCount) {
+                final byte[] key = it.key();
+                if (endKey != null && BytesUtil.compare(key, endKey) >= 0) {
+                    break;
+                }
+                entries.add(new KVEntry(key, returnValue ? it.value() : null));
+                it.next();
+            }
+            setSuccess(closure, entries);
+        } catch (final Exception e) {
+            LOG.error("Fail to [SCAN], range: ['[{}, {})'], {}.", BytesUtil.toHex(startKey), BytesUtil.toHex(endKey),
+                StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [SCAN]");
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void reverseScan(final byte[] startKey, final byte[] endKey, final int limit,
+                            @SuppressWarnings("unused") final boolean readOnlySafe, final boolean returnValue,
+                            final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("REVERSE_SCAN");
+        final List entries = Lists.newArrayList();
+        int maxCount = normalizeLimit(limit);
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (final RocksIterator it = this.db.newIterator()) {
+            if (startKey == null) {
+                it.seekToLast();
+            } else {
+                it.seekForPrev(startKey);
+            }
+            int count = 0;
+            while (it.isValid() && count++ < maxCount) {
+                final byte[] key = it.key();
+                if (endKey != null && BytesUtil.compare(key, endKey) <= 0) {
+                    break;
+                }
+                entries.add(new KVEntry(key, returnValue ? it.value() : null));
+                it.prev();
+            }
+            setSuccess(closure, entries);
+        } catch (final Exception e) {
+            LOG.error("Fail to [REVERSE_SCAN], range: ['[{}, {})'], {}.", BytesUtil.toHex(startKey),
+                BytesUtil.toHex(endKey), StackTraceUtil.stackTrace(e));
+            setFailure(closure, "Fail to [REVERSE_SCAN]");
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void getSequence(final byte[] seqKey, final int step, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET_SEQUENCE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] prevBytesVal = this.db.get(this.sequenceHandle, seqKey);
+            long startVal;
+            if (prevBytesVal == null) {
+                startVal = 0;
+            } else {
+                startVal = Bits.getLong(prevBytesVal, 0);
+            }
+            if (step < 0) {
+                // never get here
+                setFailure(closure, "Fail to [GET_SEQUENCE], step must >= 0");
+                return;
+            }
+            if (step == 0) {
+                setSuccess(closure, new Sequence(startVal, startVal));
+                return;
+            }
+            final long endVal = getSafeEndValueForSequence(startVal, step);
+            if (startVal != endVal) {
+                final byte[] newBytesVal = new byte[8];
+                Bits.putLong(newBytesVal, 0, endVal);
+                this.db.put(this.sequenceHandle, this.writeOptions, seqKey, newBytesVal);
+            }
+            setSuccess(closure, new Sequence(startVal, endVal));
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET_SEQUENCE], [key = {}, step = {}], {}.", BytesUtil.toHex(seqKey), step,
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [GET_SEQUENCE]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void resetSequence(final byte[] seqKey, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("RESET_SEQUENCE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            this.db.delete(this.sequenceHandle, seqKey);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [RESET_SEQUENCE], [key = {}], {}.", BytesUtil.toHex(seqKey),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [RESET_SEQUENCE]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchResetSequence(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            resetSequence(kvState.getOp().getKey(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_RESET_SEQUENCE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    for (final KVState kvState : segment) {
+                        batch.delete(sequenceHandle, kvState.getOp().getKey());
+                    }
+                    this.db.write(this.writeOptions, batch);
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), Boolean.TRUE);
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_RESET_SEQUENCE], [size = {}], {}.", segment.size(),
+                        StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_RESET_SEQUENCE]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void put(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            this.db.put(this.writeOptions, key, value);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [PUT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchPut(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            final KVOperation op = kvState.getOp();
+            put(op.getKey(), op.getValue(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    for (final KVState kvState : segment) {
+                        final KVOperation op = kvState.getOp();
+                        batch.put(op.getKey(), op.getValue());
+                    }
+                    this.db.write(this.writeOptions, batch);
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), Boolean.TRUE);
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_PUT], [size = {}] {}.", segment.size(), StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_PUT]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void getAndPut(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("GET_PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] prevVal = this.db.get(key);
+            this.db.put(this.writeOptions, key, value);
+            setSuccess(closure, prevVal);
+        } catch (final Exception e) {
+            LOG.error("Fail to [GET_PUT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [GET_PUT]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchGetAndPut(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            final KVOperation op = kvState.getOp();
+            getAndPut(op.getKey(), op.getValue(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_GET_PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    final List keys = Lists.newArrayListWithCapacity(segment.size());
+                    for (final KVState kvState : segment) {
+                        final KVOperation op = kvState.getOp();
+                        final byte[] key = op.getKey();
+                        keys.add(key);
+                        batch.put(key, op.getValue());
+                    }
+                    // first, get prev values
+                    final Map prevValMap = this.db.multiGet(keys);
+                    this.db.write(this.writeOptions, batch);
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), prevValMap.get(kvState.getOp().getKey()));
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_GET_PUT], [size = {}] {}.", segment.size(),
+                        StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_GET_PUT]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void compareAndPut(final byte[] key, final byte[] expect, final byte[] update, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("COMPARE_PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] actual = this.db.get(key);
+            if (Arrays.equals(expect, actual)) {
+                this.db.put(this.writeOptions, key, update);
+                setSuccess(closure, Boolean.TRUE);
+            } else {
+                setSuccess(closure, Boolean.FALSE);
+            }
+        } catch (final Exception e) {
+            LOG.error("Fail to [COMPARE_PUT], [{}, {}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(expect),
+                BytesUtil.toHex(update), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [COMPARE_PUT]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchCompareAndPut(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            final KVOperation op = kvState.getOp();
+            compareAndPut(op.getKey(), op.getExpect(), op.getValue(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_COMPARE_PUT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    final Map expects = Maps.newHashMapWithExpectedSize(segment.size());
+                    final Map updates = Maps.newHashMapWithExpectedSize(segment.size());
+                    for (final KVState kvState : segment) {
+                        final KVOperation op = kvState.getOp();
+                        final byte[] key = op.getKey();
+                        final byte[] expect = op.getExpect();
+                        final byte[] update = op.getValue();
+                        expects.put(key, expect);
+                        updates.put(key, update);
+                    }
+                    final Map prevValMap = this.db.multiGet(Lists.newArrayList(expects.keySet()));
+                    for (final KVState kvState : segment) {
+                        final byte[] key = kvState.getOp().getKey();
+                        if (Arrays.equals(expects.get(key), prevValMap.get(key))) {
+                            batch.put(key, updates.get(key));
+                            setData(kvState.getDone(), Boolean.TRUE);
+                        } else {
+                            setData(kvState.getDone(), Boolean.FALSE);
+                        }
+                    }
+                    if (batch.count() > 0) {
+                        this.db.write(this.writeOptions, batch);
+                    }
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), getData(kvState.getDone()));
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_COMPARE_PUT], [size = {}] {}.", segment.size(),
+                        StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_COMPARE_PUT]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void merge(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("MERGE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            this.db.merge(this.writeOptions, key, value);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [MERGE], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [MERGE]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchMerge(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            final KVOperation op = kvState.getOp();
+            merge(op.getKey(), op.getValue(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_MERGE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    for (final KVState kvState : segment) {
+                        final KVOperation op = kvState.getOp();
+                        batch.merge(op.getKey(), op.getValue());
+                    }
+                    this.db.write(this.writeOptions, batch);
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), Boolean.TRUE);
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_MERGE], [size = {}] {}.", segment.size(), StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_MERGE]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void put(final List entries, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT_LIST");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (final WriteBatch batch = new WriteBatch()) {
+            for (final KVEntry entry : entries) {
+                batch.put(entry.getKey(), entry.getValue());
+            }
+            this.db.write(this.writeOptions, batch);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [PUT_LIST], [size = {}], {}.", entries.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT_LIST]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void compareAndPutAll(final List entries, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("COMPARE_PUT_ALL");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (final WriteBatch batch = new WriteBatch()) {
+            final List keys = Lists.newArrayList();
+            for (final CASEntry entry : entries) {
+                keys.add(entry.getKey());
+            }
+            final Map prevValMap = this.db.multiGet(keys);
+            for (final CASEntry entry : entries) {
+                if (!Arrays.equals(entry.getExpect(), prevValMap.get(entry.getKey()))) {
+                    setSuccess(closure, Boolean.FALSE);
+                    return;
+                }
+            }
+
+            for (final CASEntry entry : entries) {
+                batch.put(entry.getKey(), entry.getUpdate());
+            }
+            if (batch.count() > 0) {
+                this.db.write(this.writeOptions, batch);
+            }
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [COMPARE_PUT_ALL], [size = {}] {}.", entries.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [COMPARE_PUT_ALL]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void putIfAbsent(final byte[] key, final byte[] value, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("PUT_IF_ABSENT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] prevVal = this.db.get(key);
+            if (prevVal == null) {
+                this.db.put(this.writeOptions, key, value);
+            }
+            setSuccess(closure, prevVal);
+        } catch (final Exception e) {
+            LOG.error("Fail to [PUT_IF_ABSENT], [{}, {}], {}.", BytesUtil.toHex(key), BytesUtil.toHex(value),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [PUT_IF_ABSENT]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchPutIfAbsent(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            final KVOperation op = kvState.getOp();
+            putIfAbsent(op.getKey(), op.getValue(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_PUT_IF_ABSENT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    final List keys = Lists.newArrayListWithCapacity(segment.size());
+                    final Map values = Maps.newHashMapWithExpectedSize(segment.size());
+                    for (final KVState kvState : segment) {
+                        final KVOperation op = kvState.getOp();
+                        final byte[] key = op.getKey();
+                        final byte[] value = op.getValue();
+                        keys.add(key);
+                        values.put(key, value);
+                    }
+                    final Map prevValMap = this.db.multiGet(keys);
+                    for (final KVState kvState : segment) {
+                        final byte[] key = kvState.getOp().getKey();
+                        final byte[] prevVal = prevValMap.get(key);
+                        if (prevVal == null) {
+                            batch.put(key, values.get(key));
+                        }
+                        setData(kvState.getDone(), prevVal);
+                    }
+                    if (batch.count() > 0) {
+                        this.db.write(this.writeOptions, batch);
+                    }
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), getData(kvState.getDone()));
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_PUT_IF_ABSENT], [size = {}] {}.", segment.size(),
+                        StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_PUT_IF_ABSENT]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void tryLockWith(final byte[] key, final byte[] fencingKey, final boolean keepLease,
+                            final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("TRY_LOCK");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            // The algorithm relies on the assumption that while there is no
+            // synchronized clock across the processes, still the local time in
+            // every process flows approximately at the same rate, with an error
+            // which is small compared to the auto-release time of the lock.
+            final long now = acquirer.getLockingTimestamp();
+            final long timeoutMillis = acquirer.getLeaseMillis();
+            final byte[] prevBytesVal = this.db.get(this.lockingHandle, key);
+
+            final DistributedLock.Owner owner;
+            // noinspection ConstantConditions
+            do {
+                final DistributedLock.OwnerBuilder builder = DistributedLock.newOwnerBuilder();
+                if (prevBytesVal == null) {
+                    // no others own this lock
+                    if (keepLease) {
+                        // it wants to keep the lease but too late, will return failure
+                        owner = builder //
+                            // set acquirer id
+                            .id(acquirer.getId())
+                            // fail to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_FAIL)
+                            // set failure
+                            .success(false).build();
+                        break;
+                    }
+                    // is first time to try lock (another possibility is that this lock has been deleted),
+                    // will return successful
+                    owner = builder //
+                        // set acquirer id, now it will own the lock
+                        .id(acquirer.getId())
+                        // set a new deadline
+                        .deadlineMillis(now + timeoutMillis)
+                        // first time to acquire and success
+                        .remainingMillis(DistributedLock.OwnerBuilder.FIRST_TIME_SUCCESS)
+                        // create a new fencing token
+                        .fencingToken(getNextFencingToken(fencingKey))
+                        // init acquires
+                        .acquires(1)
+                        // set acquirer ctx
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.db.put(this.lockingHandle, this.writeOptions, key, this.serializer.writeObject(owner));
+                    break;
+                }
+
+                // this lock has an owner, check if it has expired
+                final DistributedLock.Owner prevOwner = this.serializer.readObject(prevBytesVal,
+                    DistributedLock.Owner.class);
+                final long remainingMillis = prevOwner.getDeadlineMillis() - now;
+                if (remainingMillis < 0) {
+                    // the previous owner is out of lease
+                    if (keepLease) {
+                        // it wants to keep the lease but too late, will return failure
+                        owner = builder //
+                            // still previous owner id
+                            .id(prevOwner.getId())
+                            // do not update
+                            .deadlineMillis(prevOwner.getDeadlineMillis())
+                            // fail to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_FAIL)
+                            // set previous ctx
+                            .context(prevOwner.getContext())
+                            // set failure
+                            .success(false).build();
+                        break;
+                    }
+                    // create new lock owner
+                    owner = builder //
+                        // set acquirer id, now it will own the lock
+                        .id(acquirer.getId())
+                        // set a new deadline
+                        .deadlineMillis(now + timeoutMillis)
+                        // success as a new acquirer
+                        .remainingMillis(DistributedLock.OwnerBuilder.NEW_ACQUIRE_SUCCESS)
+                        // create a new fencing token
+                        .fencingToken(getNextFencingToken(fencingKey))
+                        // init acquires
+                        .acquires(1)
+                        // set acquirer ctx
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.db.put(this.lockingHandle, this.writeOptions, key, this.serializer.writeObject(owner));
+                    break;
+                }
+
+                // the previous owner is not out of lease (remainingMillis >= 0)
+                final boolean isReentrant = prevOwner.isSameAcquirer(acquirer);
+                if (isReentrant) {
+                    // is the same old friend come back (reentrant lock)
+                    if (keepLease) {
+                        // the old friend only wants to keep lease of lock
+                        owner = builder //
+                            // still previous owner id
+                            .id(prevOwner.getId())
+                            // update the deadline to keep lease
+                            .deadlineMillis(now + timeoutMillis)
+                            // success to keep lease
+                            .remainingMillis(DistributedLock.OwnerBuilder.KEEP_LEASE_SUCCESS)
+                            // keep fencing token
+                            .fencingToken(prevOwner.getFencingToken())
+                            // keep acquires
+                            .acquires(prevOwner.getAcquires())
+                            // do not update ctx when keeping lease
+                            .context(prevOwner.getContext())
+                            // set successful
+                            .success(true).build();
+                        this.db.put(this.lockingHandle, this.writeOptions, key, this.serializer.writeObject(owner));
+                        break;
+                    }
+                    // now we are sure that is an old friend who is back again (reentrant lock)
+                    owner = builder //
+                        // still previous owner id
+                        .id(prevOwner.getId())
+                        // by the way, the lease will also be kept
+                        .deadlineMillis(now + timeoutMillis)
+                        // success reentrant
+                        .remainingMillis(DistributedLock.OwnerBuilder.REENTRANT_SUCCESS)
+                        // keep fencing token
+                        .fencingToken(prevOwner.getFencingToken())
+                        // acquires++
+                        .acquires(prevOwner.getAcquires() + 1)
+                        // update ctx when reentrant
+                        .context(acquirer.getContext())
+                        // set successful
+                        .success(true).build();
+                    this.db.put(this.lockingHandle, this.writeOptions, key, this.serializer.writeObject(owner));
+                    break;
+                }
+
+                // the lock is exist and also prev locker is not the same as current
+                owner = builder //
+                    // set previous owner id to tell who is the real owner
+                    .id(prevOwner.getId())
+                    // set the remaining lease time of current owner
+                    .remainingMillis(remainingMillis)
+                    // set previous ctx
+                    .context(prevOwner.getContext())
+                    // set failure
+                    .success(false).build();
+                LOG.debug("Another locker [{}] is trying the existed lock [{}].", acquirer, prevOwner);
+            } while (false);
+
+            setSuccess(closure, owner);
+        } catch (final Exception e) {
+            LOG.error("Fail to [TRY_LOCK], [{}, {}], {}.", BytesUtil.toHex(key), acquirer, StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [TRY_LOCK]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void releaseLockWith(final byte[] key, final DistributedLock.Acquirer acquirer, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("RELEASE_LOCK");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] prevBytesVal = this.db.get(this.lockingHandle, key);
+
+            final DistributedLock.Owner owner;
+            // noinspection ConstantConditions
+            do {
+                final DistributedLock.OwnerBuilder builder = DistributedLock.newOwnerBuilder();
+                if (prevBytesVal == null) {
+                    LOG.warn("Lock not exist: {}.", acquirer);
+                    owner = builder //
+                        // set acquirer id
+                        .id(acquirer.getId())
+                        // set acquirer fencing token
+                        .fencingToken(acquirer.getFencingToken())
+                        // set acquires=0
+                        .acquires(0)
+                        // set successful
+                        .success(true).build();
+                    break;
+                }
+
+                final DistributedLock.Owner prevOwner = this.serializer.readObject(prevBytesVal,
+                    DistributedLock.Owner.class);
+
+                if (prevOwner.isSameAcquirer(acquirer)) {
+                    final long acquires = prevOwner.getAcquires() - 1;
+                    owner = builder //
+                        // still previous owner id
+                        .id(prevOwner.getId())
+                        // do not update deadline
+                        .deadlineMillis(prevOwner.getDeadlineMillis())
+                        // keep fencing token
+                        .fencingToken(prevOwner.getFencingToken())
+                        // acquires--
+                        .acquires(acquires)
+                        // set previous ctx
+                        .context(prevOwner.getContext())
+                        // set successful
+                        .success(true).build();
+                    if (acquires <= 0) {
+                        // real delete, goodbye ~
+                        this.db.delete(this.lockingHandle, this.writeOptions, key);
+                    } else {
+                        // acquires--
+                        this.db.put(this.lockingHandle, this.writeOptions, key, this.serializer.writeObject(owner));
+                    }
+                    break;
+                }
+
+                // invalid acquirer, can't to release the lock
+                owner = builder //
+                    // set previous owner id to tell who is the real owner
+                    .id(prevOwner.getId())
+                    // keep previous fencing token
+                    .fencingToken(prevOwner.getFencingToken())
+                    // do not update acquires
+                    .acquires(prevOwner.getAcquires())
+                    // set previous ctx
+                    .context(prevOwner.getContext())
+                    // set failure
+                    .success(false).build();
+                LOG.warn("The lock owner is: [{}], [{}] could't release it.", prevOwner, acquirer);
+            } while (false);
+
+            setSuccess(closure, owner);
+        } catch (final Exception e) {
+            LOG.error("Fail to [RELEASE_LOCK], [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [RELEASE_LOCK]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    private long getNextFencingToken(final byte[] fencingKey) throws RocksDBException {
+        final Timer.Context timeCtx = getTimeContext("FENCING_TOKEN");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] realKey = BytesUtil.nullToEmpty(fencingKey);
+            final byte[] prevBytesVal = this.db.get(this.fencingHandle, realKey);
+            final long prevVal;
+            if (prevBytesVal == null) {
+                prevVal = 0; // init
+            } else {
+                prevVal = Bits.getLong(prevBytesVal, 0);
+            }
+            // Don't worry about the token number overflow.
+            // It takes about 290,000 years for the 1 million TPS system
+            // to use the numbers in the range [0 ~ Long.MAX_VALUE].
+            final long newVal = prevVal + 1;
+            final byte[] newBytesVal = new byte[8];
+            Bits.putLong(newBytesVal, 0, newVal);
+            this.db.put(this.fencingHandle, this.writeOptions, realKey, newBytesVal);
+            return newVal;
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void delete(final byte[] key, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            this.db.delete(this.writeOptions, key);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [DELETE], [{}], {}.", BytesUtil.toHex(key), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void batchDelete(final KVStateOutputList kvStates) {
+        if (kvStates.isSingletonList()) {
+            final KVState kvState = kvStates.getSingletonElement();
+            delete(kvState.getOp().getKey(), kvState.getDone());
+            return;
+        }
+        final Timer.Context timeCtx = getTimeContext("BATCH_DELETE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            Partitions.manyToOne(kvStates, MAX_BATCH_WRITE_SIZE, (Function, Void>) segment -> {
+                try (final WriteBatch batch = new WriteBatch()) {
+                    for (final KVState kvState : segment) {
+                        batch.delete(kvState.getOp().getKey());
+                    }
+                    this.db.write(this.writeOptions, batch);
+                    for (final KVState kvState : segment) {
+                        setSuccess(kvState.getDone(), Boolean.TRUE);
+                    }
+                } catch (final Exception e) {
+                    LOG.error("Failed to [BATCH_DELETE], [size = {}], {}.", segment.size(),
+                        StackTraceUtil.stackTrace(e));
+                    setCriticalError(Lists.transform(kvStates, KVState::getDone), "Fail to [BATCH_DELETE]", e);
+                }
+                return null;
+            });
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void deleteRange(final byte[] startKey, final byte[] endKey, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE_RANGE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            this.db.deleteRange(this.writeOptions, startKey, endKey);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Fail to [DELETE_RANGE], ['[{}, {})'], {}.", BytesUtil.toHex(startKey), BytesUtil.toHex(endKey),
+                StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE_RANGE]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void delete(final List keys, final KVStoreClosure closure) {
+        final Timer.Context timeCtx = getTimeContext("DELETE_LIST");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try (final WriteBatch batch = new WriteBatch()) {
+            for (final byte[] key : keys) {
+                batch.delete(key);
+            }
+            this.db.write(this.writeOptions, batch);
+            setSuccess(closure, Boolean.TRUE);
+        } catch (final Exception e) {
+            LOG.error("Failed to [DELETE_LIST], [size = {}], {}.", keys.size(), StackTraceUtil.stackTrace(e));
+            setCriticalError(closure, "Fail to [DELETE_LIST]", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public long getApproximateKeysInRange(final byte[] startKey, final byte[] endKey) {
+        // TODO This is a sad code, the performance is too damn bad
+        final Timer.Context timeCtx = getTimeContext("APPROXIMATE_KEYS");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        final Snapshot snapshot = this.db.getSnapshot();
+        try (final ReadOptions readOptions = new ReadOptions()) {
+            readOptions.setSnapshot(snapshot);
+            try (final RocksIterator it = this.db.newIterator(readOptions)) {
+                if (startKey == null) {
+                    it.seekToFirst();
+                } else {
+                    it.seek(startKey);
+                }
+                long approximateKeys = 0;
+                for (;;) {
+                    // The accuracy is 100, don't ask more
+                    for (int i = 0; i < 100; i++) {
+                        if (!it.isValid()) {
+                            return approximateKeys;
+                        }
+                        it.next();
+                        ++approximateKeys;
+                    }
+                    if (endKey != null && BytesUtil.compare(it.key(), endKey) >= 0) {
+                        return approximateKeys;
+                    }
+                }
+            }
+        } finally {
+            // Nothing to release, rocksDB never own the pointer for a snapshot.
+            snapshot.close();
+            // The pointer to the snapshot is released by the database instance.
+            this.db.releaseSnapshot(snapshot);
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public byte[] jumpOver(final byte[] startKey, final long distance) {
+        final Timer.Context timeCtx = getTimeContext("JUMP_OVER");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        final Snapshot snapshot = this.db.getSnapshot();
+        try (final ReadOptions readOptions = new ReadOptions()) {
+            readOptions.setSnapshot(snapshot);
+            try (final RocksIterator it = this.db.newIterator(readOptions)) {
+                if (startKey == null) {
+                    it.seekToFirst();
+                } else {
+                    it.seek(startKey);
+                }
+                long approximateKeys = 0;
+                for (;;) {
+                    byte[] lastKey = null;
+                    if (it.isValid()) {
+                        lastKey = it.key();
+                    }
+                    // The accuracy is 100, don't ask more
+                    for (int i = 0; i < 100; i++) {
+                        if (!it.isValid()) {
+                            return lastKey;
+                        }
+                        it.next();
+                        if (++approximateKeys >= distance) {
+                            return it.key();
+                        }
+                    }
+                }
+            }
+        } finally {
+            // Nothing to release, rocksDB never own the pointer for a snapshot.
+            snapshot.close();
+            // The pointer to the snapshot is released by the database instance.
+            this.db.releaseSnapshot(snapshot);
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    @Override
+    public void initFencingToken(final byte[] parentKey, final byte[] childKey) {
+        final Timer.Context timeCtx = getTimeContext("INIT_FENCING_TOKEN");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final byte[] realKey = BytesUtil.nullToEmpty(parentKey);
+            final byte[] parentBytesVal = this.db.get(this.fencingHandle, realKey);
+            if (parentBytesVal == null) {
+                return;
+            }
+            this.db.put(this.fencingHandle, this.writeOptions, childKey, parentBytesVal);
+        } catch (final RocksDBException e) {
+            throw new StorageException("Fail to init fencing token.", e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    public long getDatabaseVersion() {
+        return this.databaseVersion.get();
+    }
+
+    public void addStatisticsCollectorCallback(final StatisticsCollectorCallback callback) {
+        final RocksStatisticsCollector collector = Requires.requireNonNull(this.statisticsCollector,
+            "statisticsCollector");
+        final Statistics statistics = Requires.requireNonNull(this.statistics, "statistics");
+        collector.addStatsCollectorInput(new StatsCollectorInput(statistics, callback));
+    }
+
+    boolean isFastSnapshot() {
+        return Requires.requireNonNull(this.opts, "opts").isFastSnapshot();
+    }
+
+    boolean isAsyncSnapshot() {
+        return Requires.requireNonNull(this.opts, "opts").isAsyncSnapshot();
+    }
+
+    CompletableFuture createSstFiles(final EnumMap sstFileTable, final byte[] startKey,
+                                           final byte[] endKey, final ExecutorService executor) {
+        final Snapshot snapshot;
+        final CompletableFuture sstFuture = new CompletableFuture<>();
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            snapshot = this.db.getSnapshot();
+            if (!isAsyncSnapshot()) {
+                doCreateSstFiles(snapshot, sstFileTable, startKey, endKey, sstFuture);
+                return sstFuture;
+            }
+        } finally {
+            readLock.unlock();
+        }
+
+        // async snapshot
+        executor.execute(() -> doCreateSstFiles(snapshot, sstFileTable, startKey, endKey, sstFuture));
+        return sstFuture;
+    }
+
+    void doCreateSstFiles(final Snapshot snapshot, final EnumMap sstFileTable,
+                          final byte[] startKey, final byte[] endKey, final CompletableFuture future) {
+        final Timer.Context timeCtx = getTimeContext("CREATE_SST_FILE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            if (!this.shutdownLock.isAvailable()) {
+                // KV store has shutdown, we do not release rocksdb's snapshot
+                future.completeExceptionally(new StorageException("KV store has shutdown."));
+                return;
+            }
+            try (final ReadOptions readOptions = new ReadOptions();
+                    final EnvOptions envOptions = new EnvOptions();
+                    final Options options = new Options().setMergeOperator(new StringAppendOperator())) {
+                readOptions.setSnapshot(snapshot);
+                for (final Map.Entry entry : sstFileTable.entrySet()) {
+                    final SstColumnFamily sstColumnFamily = entry.getKey();
+                    final File sstFile = entry.getValue();
+                    final ColumnFamilyHandle columnFamilyHandle = findColumnFamilyHandle(sstColumnFamily);
+                    try (final RocksIterator it = this.db.newIterator(columnFamilyHandle, readOptions);
+                            final SstFileWriter sstFileWriter = new SstFileWriter(envOptions, options)) {
+                        if (startKey == null) {
+                            it.seekToFirst();
+                        } else {
+                            it.seek(startKey);
+                        }
+                        sstFileWriter.open(sstFile.getAbsolutePath());
+                        long count = 0;
+                        for (;;) {
+                            if (!it.isValid()) {
+                                break;
+                            }
+                            final byte[] key = it.key();
+                            if (endKey != null && BytesUtil.compare(key, endKey) >= 0) {
+                                break;
+                            }
+                            sstFileWriter.put(key, it.value());
+                            ++count;
+                            it.next();
+                        }
+                        if (count == 0) {
+                            sstFileWriter.close();
+                        } else {
+                            sstFileWriter.finish();
+                        }
+                        LOG.info("Finish sst file {} with {} keys.", sstFile, count);
+                    } catch (final RocksDBException e) {
+                        throw new StorageException("Fail to create sst file at path: " + sstFile, e);
+                    }
+                }
+                future.complete(null);
+            } catch (final Throwable t) {
+                future.completeExceptionally(t);
+            } finally {
+                // Nothing to release, rocksDB never own the pointer for a snapshot.
+                snapshot.close();
+                // The pointer to the snapshot is released by the database instance.
+                this.db.releaseSnapshot(snapshot);
+            }
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    void ingestSstFiles(final EnumMap sstFileTable) {
+        final Timer.Context timeCtx = getTimeContext("INGEST_SST_FILE");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            for (final Map.Entry entry : sstFileTable.entrySet()) {
+                final SstColumnFamily sstColumnFamily = entry.getKey();
+                final File sstFile = entry.getValue();
+                final ColumnFamilyHandle columnFamilyHandle = findColumnFamilyHandle(sstColumnFamily);
+                try (final IngestExternalFileOptions ingestOptions = new IngestExternalFileOptions()) {
+                    if (FileUtils.sizeOf(sstFile) == 0L) {
+                        return;
+                    }
+                    final String filePath = sstFile.getAbsolutePath();
+                    LOG.info("Start ingest sst file {}.", filePath);
+                    this.db.ingestExternalFile(columnFamilyHandle, Collections.singletonList(filePath), ingestOptions);
+                } catch (final RocksDBException e) {
+                    throw new StorageException("Fail to ingest sst file at path: " + sstFile, e);
+                }
+            }
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    RocksDBBackupInfo backupDB(final String backupDBPath) throws IOException {
+        final Timer.Context timeCtx = getTimeContext("BACKUP_DB");
+        FileUtils.forceMkdir(new File(backupDBPath));
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try (final BackupableDBOptions backupOpts = createBackupDBOptions(backupDBPath);
+             final BackupEngine backupEngine = BackupEngine.open(this.options.getEnv(), backupOpts)) {
+            backupEngine.createNewBackup(this.db, true);
+            final List backupInfoList = backupEngine.getBackupInfo();
+            if (backupInfoList.isEmpty()) {
+                LOG.warn("Fail to backup at {}, empty backup info.", backupDBPath);
+                return null;
+            }
+            // chose the backupInfo who has max backupId
+            final BackupInfo backupInfo = Collections.max(backupInfoList, Comparator.comparingInt(BackupInfo::backupId));
+            final RocksDBBackupInfo rocksBackupInfo = new RocksDBBackupInfo(backupInfo);
+            LOG.info("Backup rocksDB into {} with backupInfo {}.", backupDBPath, rocksBackupInfo);
+            return rocksBackupInfo;
+        } catch (final RocksDBException e) {
+            throw new StorageException("Fail to backup at path: " + backupDBPath, e);
+        } finally {
+            writeLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    void restoreBackup(final String backupDBPath, final RocksDBBackupInfo rocksBackupInfo) {
+        final Timer.Context timeCtx = getTimeContext("RESTORE_BACKUP");
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        closeRocksDB();
+        try (final BackupableDBOptions backupOpts = createBackupDBOptions(backupDBPath);
+                final BackupEngine backupEngine = BackupEngine.open(this.options.getEnv(), backupOpts);
+                final RestoreOptions restoreOpts = new RestoreOptions(false)) {
+            final String dbPath = this.opts.getDbPath();
+            backupEngine.restoreDbFromBackup(rocksBackupInfo.getBackupId(), dbPath, dbPath, restoreOpts);
+            LOG.info("Restored rocksDB from {} with {}.", backupDBPath, rocksBackupInfo);
+            // reopen the db
+            openRocksDB(this.opts);
+        } catch (final RocksDBException e) {
+            throw new StorageException("Fail to restore from path: " + backupDBPath, e);
+        } finally {
+            writeLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    void writeSnapshot(final String snapshotPath) {
+        final Timer.Context timeCtx = getTimeContext("WRITE_SNAPSHOT");
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try (final Checkpoint checkpoint = Checkpoint.create(this.db)) {
+            final String tempPath = snapshotPath + "_temp";
+            final File tempFile = new File(tempPath);
+            FileUtils.deleteDirectory(tempFile);
+            checkpoint.createCheckpoint(tempPath);
+            final File snapshotFile = new File(snapshotPath);
+            FileUtils.deleteDirectory(snapshotFile);
+            if (!Utils.atomicMoveFile(tempFile, snapshotFile, true)) {
+                throw new StorageException("Fail to rename [" + tempPath + "] to [" + snapshotPath + "].");
+            }
+        } catch (final StorageException e) {
+            throw e;
+        } catch (final Exception e) {
+            throw new StorageException("Fail to write snapshot at path: " + snapshotPath, e);
+        } finally {
+            writeLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    void readSnapshot(final String snapshotPath) {
+        final Timer.Context timeCtx = getTimeContext("READ_SNAPSHOT");
+        final Lock writeLock = this.readWriteLock.writeLock();
+        writeLock.lock();
+        try {
+            final File snapshotFile = new File(snapshotPath);
+            if (!snapshotFile.exists()) {
+                LOG.error("Snapshot file [{}] not exists.", snapshotPath);
+                return;
+            }
+            closeRocksDB();
+            final String dbPath = this.opts.getDbPath();
+            final File dbFile = new File(dbPath);
+            FileUtils.deleteDirectory(dbFile);
+            if (!Utils.atomicMoveFile(snapshotFile, dbFile, true)) {
+                throw new StorageException("Fail to rename [" + snapshotPath + "] to [" + dbPath + "].");
+            }
+            // reopen the db
+            openRocksDB(this.opts);
+        } catch (final Exception e) {
+            throw new StorageException("Fail to read snapshot from path: " + snapshotPath, e);
+        } finally {
+            writeLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    CompletableFuture writeSstSnapshot(final String snapshotPath, final Region region, final ExecutorService executor) {
+        final Timer.Context timeCtx = getTimeContext("WRITE_SST_SNAPSHOT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final String tempPath = snapshotPath + "_temp";
+            final File tempFile = new File(tempPath);
+            FileUtils.deleteDirectory(tempFile);
+            FileUtils.forceMkdir(tempFile);
+
+            final EnumMap sstFileTable = getSstFileTable(tempPath);
+            final CompletableFuture snapshotFuture = new CompletableFuture<>();
+            final CompletableFuture sstFuture = createSstFiles(sstFileTable, region.getStartKey(),
+                region.getEndKey(), executor);
+            sstFuture.whenComplete((aVoid, throwable) -> {
+                if (throwable == null) {
+                    try {
+                        final File snapshotFile = new File(snapshotPath);
+                        FileUtils.deleteDirectory(snapshotFile);
+                        if (!Utils.atomicMoveFile(tempFile, snapshotFile, true)) {
+                            throw new StorageException("Fail to rename [" + tempPath + "] to [" + snapshotPath + "].");
+                        }
+                        snapshotFuture.complete(null);
+                    } catch (final Throwable t) {
+                        snapshotFuture.completeExceptionally(t);
+                    }
+                } else {
+                    snapshotFuture.completeExceptionally(throwable);
+                }
+            });
+            return snapshotFuture;
+        } catch (final Exception e) {
+            throw new StorageException("Fail to do read sst snapshot at path: " + snapshotPath, e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    void readSstSnapshot(final String snapshotPath) {
+        final Timer.Context timeCtx = getTimeContext("READ_SST_SNAPSHOT");
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            final EnumMap sstFileTable = getSstFileTable(snapshotPath);
+            ingestSstFiles(sstFileTable);
+        } catch (final Exception e) {
+            throw new StorageException("Fail to write sst snapshot at path: " + snapshotPath, e);
+        } finally {
+            readLock.unlock();
+            timeCtx.stop();
+        }
+    }
+
+    private EnumMap getSstFileTable(final String path) {
+        final EnumMap sstFileTable = new EnumMap<>(SstColumnFamily.class);
+        sstFileTable.put(SstColumnFamily.DEFAULT, Paths.get(path, "default.sst").toFile());
+        sstFileTable.put(SstColumnFamily.SEQUENCE, Paths.get(path, "sequence.sst").toFile());
+        sstFileTable.put(SstColumnFamily.LOCKING, Paths.get(path, "locking.sst").toFile());
+        sstFileTable.put(SstColumnFamily.FENCING, Paths.get(path, "fencing.sst").toFile());
+        return sstFileTable;
+    }
+
+    private ColumnFamilyHandle findColumnFamilyHandle(final SstColumnFamily sstColumnFamily) {
+        switch (sstColumnFamily) {
+            case DEFAULT:
+                return this.defaultHandle;
+            case SEQUENCE:
+                return this.sequenceHandle;
+            case LOCKING:
+                return this.lockingHandle;
+            case FENCING:
+                return this.fencingHandle;
+            default:
+                throw new IllegalArgumentException("illegal sstColumnFamily: " + sstColumnFamily.name());
+        }
+    }
+
+    private void openRocksDB(final RocksDBOptions opts) throws RocksDBException {
+        final List cfHandles = Lists.newArrayList();
+        this.databaseVersion.incrementAndGet();
+        this.db = RocksDB.open(this.options, opts.getDbPath(), this.cfDescriptors, cfHandles);
+        this.defaultHandle = cfHandles.get(0);
+        this.sequenceHandle = cfHandles.get(1);
+        this.lockingHandle = cfHandles.get(2);
+        this.fencingHandle = cfHandles.get(3);
+    }
+
+    private void closeRocksDB() {
+        if (this.db != null) {
+            this.db.close();
+            this.db = null;
+        }
+    }
+
+    private void destroyRocksDB(final RocksDBOptions opts) throws RocksDBException {
+        // The major difference with directly deleting the DB directory manually is that
+        // DestroyDB() will take care of the case where the RocksDB database is stored
+        // in multiple directories. For instance, a single DB can be configured to store
+        // its data in multiple directories by specifying different paths to
+        // DBOptions::db_paths, DBOptions::db_log_dir, and DBOptions::wal_dir.
+        try (final Options opt = new Options()) {
+            RocksDB.destroyDB(opts.getDbPath(), opt);
+        }
+    }
+
+    // Creates the rocksDB options, the user must take care
+    // to close it after closing db.
+    private static DBOptions createDBOptions() {
+        return StorageOptionsFactory.getRocksDBOptions(RocksRawKVStore.class) //
+            .setEnv(Env.getDefault());
+    }
+
+    // Creates the column family options to control the behavior
+    // of a database.
+    private static ColumnFamilyOptions createColumnFamilyOptions() {
+        final BlockBasedTableConfig tConfig = StorageOptionsFactory.getRocksDBTableFormatConfig(RocksRawKVStore.class);
+        return StorageOptionsFactory.getRocksDBColumnFamilyOptions(RocksRawKVStore.class) //
+            .setTableFormatConfig(tConfig) //
+            .setMergeOperator(new StringAppendOperator());
+    }
+
+    // Creates the backupable db options to control the behavior of
+    // a backupable database.
+    private static BackupableDBOptions createBackupDBOptions(final String backupDBPath) {
+        return new BackupableDBOptions(backupDBPath) //
+            .setSync(true) //
+            .setShareTableFiles(false); // don't share data between backups
+    }
+
+    @Override
+    public void describe(final Printer out) {
+        final Lock readLock = this.readWriteLock.readLock();
+        readLock.lock();
+        try {
+            if (this.db != null) {
+                out.println(this.db.getProperty("rocksdb.stats"));
+            }
+            out.println("");
+            if (this.statistics != null) {
+                out.println(this.statistics.getString());
+            }
+        } catch (final RocksDBException e) {
+            out.println(e);
+        } finally {
+            readLock.unlock();
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/Sequence.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/Sequence.java
new file mode 100644
index 0000000..c3902bc
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/Sequence.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.Serializable;
+
+/**
+ * Self-increasing sequence, each value is globally unique,
+ * range: [startValue, endValue)
+ *
+ * @author jiachun.fjc
+ */
+public class Sequence implements Serializable {
+
+    private static final long serialVersionUID = 25761738530535127L;
+
+    private final long        startValue;                           // inclusive
+    private final long        endValue;                             // exclusive
+
+    public Sequence(long startValue, long endValue) {
+        this.startValue = startValue;
+        this.endValue = endValue;
+    }
+
+    /**
+     * The available minimum value, including [startValue, endValue)
+     */
+    public long getStartValue() {
+        return startValue;
+    }
+
+    /**
+     * The available maximum value, not including [startValue, endValue)
+     */
+    public long getEndValue() {
+        return endValue;
+    }
+
+    @Override
+    public String toString() {
+        return "Sequence{" + "startValue=" + startValue + ", endValue=" + endValue + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/SstColumnFamily.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/SstColumnFamily.java
new file mode 100644
index 0000000..2d36474
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/SstColumnFamily.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ * @author jiachun.fjc
+ */
+public enum SstColumnFamily {
+
+    DEFAULT(0), SEQUENCE(1), LOCKING(2), FENCING(3);
+
+    private final int value;
+
+    SstColumnFamily(int value) {
+        this.value = value;
+    }
+
+    public int getValue() {
+        return value;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/StorageType.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/StorageType.java
new file mode 100644
index 0000000..142fc5d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/StorageType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ * @author jiachun.fjc
+ */
+public enum StorageType {
+    RocksDB, Memory
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategy.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategy.java
new file mode 100644
index 0000000..aa4391e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategy.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.rhea.util.ZipUtil;
+import java.util.zip.Checksum;
+
+/**
+ * @author hzh
+ */
+public class JDKZipStrategy implements ZipStrategy {
+
+    @Override
+    public void compress(final String rootDir, final String sourceDir, final String outputZipFile,
+                         final Checksum checksum) throws Throwable {
+        ZipUtil.compress(rootDir, sourceDir, outputZipFile, checksum);
+    }
+
+    @Override
+    public void deCompress(final String sourceZipFile, final String outputDir, final Checksum checksum)
+                                                                                                       throws Throwable {
+        ZipUtil.decompress(sourceZipFile, outputDir, checksum);
+    }
+
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategy.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategy.java
new file mode 100644
index 0000000..005b5ac
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategy.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+import com.alipay.sofa.jraft.util.Utils;
+import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.archivers.zip.ZipFile;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.input.NullInputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.InputStream;
+import java.nio.file.Paths;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.Checksum;
+import java.util.zip.ZipEntry;
+
+/**
+ * @author hzh
+ */
+public class ParallelZipStrategy implements ZipStrategy {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ParallelZipStrategy.class);
+    private final int           compressThreads;
+    private final int           deCompressThreads;
+
+    public ParallelZipStrategy(final int compressThreads, final int deCompressThreads) {
+        this.compressThreads = compressThreads;
+        this.deCompressThreads = deCompressThreads;
+    }
+
+    /**
+     * Parallel output streams controller
+     */
+    private static class ZipArchiveScatterOutputStream {
+
+        private final ParallelScatterZipCreator creator;
+
+        public ZipArchiveScatterOutputStream(final ExecutorService executorService) {
+            this.creator = new ParallelScatterZipCreator(executorService);
+        }
+
+        public void addEntry(final ZipArchiveEntry entry, final InputStreamSupplier supplier) {
+            creator.addArchiveEntry(entry, supplier);
+        }
+
+        public void writeTo(final ZipArchiveOutputStream archiveOutput) throws Exception {
+            creator.writeTo(archiveOutput);
+        }
+
+    }
+
+    @Override
+    public void compress(final String rootDir, final String sourceDir, final String outputZipFile,
+                         final Checksum checksum) throws Throwable {
+        final File rootFile = new File(Paths.get(rootDir, sourceDir).toString());
+        final File zipFile = new File(outputZipFile);
+        FileUtils.forceMkdir(zipFile.getParentFile());
+
+        // parallel compress
+        final ExecutorService compressExecutor = newFixedPool(this.compressThreads, "rheakv-raft-compress-executor");
+        final ZipArchiveScatterOutputStream scatterOutput = new ZipArchiveScatterOutputStream(compressExecutor);
+        compressDirectoryToZipFile(rootFile, scatterOutput, sourceDir, ZipEntry.DEFLATED);
+
+        // write and flush
+        try (final FileOutputStream fos = new FileOutputStream(zipFile);
+                final BufferedOutputStream bos = new BufferedOutputStream(fos);
+                final CheckedOutputStream cos = new CheckedOutputStream(bos, checksum);
+                final ZipArchiveOutputStream archiveOutputStream = new ZipArchiveOutputStream(cos)) {
+            scatterOutput.writeTo(archiveOutputStream);
+            archiveOutputStream.flush();
+            fos.getFD().sync();
+        }
+        Utils.fsync(zipFile);
+        // shutdown executor
+        ExecutorServiceHelper.shutdownAndAwaitTermination(compressExecutor);
+    }
+
+    @Override
+    public void deCompress(final String sourceZipFile, final String outputDir, final Checksum checksum) throws Throwable {
+        final ExecutorService deCompressExecutor = newFixedPool(this.deCompressThreads, "rheakv-raft-decompress-executor");
+        // compute the checksum in a single thread
+        final Future checksumFuture = deCompressExecutor.submit(() -> {
+            computeZipFileChecksumValue(sourceZipFile, checksum);
+            return true;
+        });
+        // decompress zip file in thread pool
+        try (final ZipFile zipFile = new ZipFile(sourceZipFile)) {
+            final List> futures = Lists.newArrayList();
+            for (final Enumeration e = zipFile.getEntries(); e.hasMoreElements(); ) {
+                final ZipArchiveEntry zipEntry = e.nextElement();
+                final Future future = deCompressExecutor.submit(() -> {
+                    unZipFile(zipFile, zipEntry, outputDir);
+                    return true;
+                });
+                futures.add(future);
+            }
+            // blocking and caching exception
+            for (final Future future : futures) {
+                future.get();
+            }
+        }
+        // wait for checksum to be calculated
+        checksumFuture.get();
+        // shutdown executor
+        ExecutorServiceHelper.shutdownAndAwaitTermination(deCompressExecutor);
+    }
+
+    private void compressDirectoryToZipFile(final File dir, final ZipArchiveScatterOutputStream scatterOutput,
+                                            final String sourceDir, final int method) {
+        if (dir == null) {
+            return;
+        }
+        if (dir.isFile()) {
+            addEntry(sourceDir, dir, scatterOutput, method);
+            return;
+        }
+        final File[] files = Requires.requireNonNull(dir.listFiles(), "files");
+        for (final File file : files) {
+            final String child = Paths.get(sourceDir, file.getName()).toString();
+            if (file.isDirectory()) {
+                compressDirectoryToZipFile(file, scatterOutput, child, method);
+            } else {
+                addEntry(child, file, scatterOutput, method);
+            }
+        }
+    }
+
+    /**
+     * Add archive entry to the scatterOutputStream
+     */
+    private void addEntry(final String filePath, final File file, final ZipArchiveScatterOutputStream scatterOutputStream, final int method) {
+        final ZipArchiveEntry archiveEntry = new ZipArchiveEntry(filePath);
+        archiveEntry.setMethod(method);
+        scatterOutputStream.addEntry(archiveEntry, () -> {
+            try {
+                return file.isDirectory() ? new NullInputStream(0) :
+                                                    new BufferedInputStream(new FileInputStream(file));
+            } catch (final FileNotFoundException e) {
+                LOG.error("Can't find file, path={}, {}", file.getPath(), StackTraceUtil.stackTrace(e));
+            }
+            return new NullInputStream(0) ;
+        });
+    }
+
+    /**
+     * Unzip the archive entry to targetDir
+     */
+    private void unZipFile(final ZipFile zipFile, final ZipArchiveEntry entry, final String targetDir) throws Exception {
+        final File targetFile = new File(Paths.get(targetDir, entry.getName()).toString());
+        FileUtils.forceMkdir(targetFile.getParentFile());
+        try (final InputStream is = zipFile.getInputStream(entry);
+                final BufferedInputStream fis = new BufferedInputStream(is);
+                final BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(targetFile))) {
+            IOUtils.copy(fis, bos);
+        }
+    }
+
+    /**
+     * Compute the value of checksum
+     */
+    private void computeZipFileChecksumValue(final String zipPath, final Checksum checksum) throws Exception {
+        try (final BufferedInputStream bis = new BufferedInputStream(new FileInputStream(zipPath));
+                final CheckedInputStream cis = new CheckedInputStream(bis, checksum);
+                final ZipArchiveInputStream zis = new ZipArchiveInputStream(cis)) {
+            // checksum is calculated in the process
+            while ((zis.getNextZipEntry()) != null)
+                ;
+        }
+    }
+
+    private static ExecutorService newFixedPool(final int coreThreads, final String poolName) {
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(poolName) //
+            .enableMetric(true) //
+            .coreThreads(coreThreads) //
+            .maximumThreads(coreThreads) //
+            .keepAliveSeconds(60L) //
+            .workQueue(new LinkedBlockingQueue<>()) //
+            .threadFactory(new NamedThreadFactory(poolName, true)) //
+            .rejectedHandler(new CallerRunsPolicyWithReport(poolName, poolName)) //
+            .build();
+    }
+
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategy.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategy.java
new file mode 100644
index 0000000..3587ab6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategy.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import java.util.zip.Checksum;
+
+/**
+ * @author hzh
+ */
+public interface ZipStrategy {
+
+    /**
+     * Compress files to zip
+     *
+     * @param rootDir    the origin file root dir
+     * @param sourceDir  the origin file source dir
+     * @param outputZipFile the target zip file
+     * @param checksum   checksum
+     */
+    void compress(final String rootDir, final String sourceDir, final String outputZipFile, final Checksum checksum)
+                                                                                                                    throws Throwable;
+
+    /**
+     * Decompress zip to files
+     *
+     * @param sourceZipFile the origin zip file
+     * @param outputDir  the target file dir
+     * @param checksum   checksum
+     */
+    void deCompress(final String sourceZipFile, final String outputDir, final Checksum checksum) throws Throwable;
+
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManager.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManager.java
new file mode 100644
index 0000000..eee7d63
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManager.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+
+/**
+ * @author hzh
+ */
+public final class ZipStrategyManager {
+    private static ZipStrategy[] zipStrategies     = new ZipStrategy[5];
+    private static byte          DEFAULT_STRATEGY  = 1;
+    public static final byte     JDK_STRATEGY      = 1;
+    public static final byte     PARALLEL_STRATEGY = 2;
+
+    static {
+        addZipStrategy(JDK_STRATEGY, new JDKZipStrategy());
+    }
+
+    public static void addZipStrategy(final int idx, final ZipStrategy zipStrategy) {
+        if (zipStrategies.length <= idx) {
+            final ZipStrategy[] newZipStrategies = new ZipStrategy[idx + 5];
+            System.arraycopy(zipStrategies, 0, newZipStrategies, 0, zipStrategies.length);
+            zipStrategies = newZipStrategies;
+        }
+        zipStrategies[idx] = zipStrategy;
+    }
+
+    public static ZipStrategy getZipStrategy(final int idx) {
+        return zipStrategies[idx];
+    }
+
+    public static ZipStrategy getDefault() {
+        return zipStrategies[DEFAULT_STRATEGY];
+    }
+
+    public static void init(final RheaKVStoreOptions opts) {
+        // add parallel zip strategy
+        if (opts.isUseParallelCompress()) {
+            if (zipStrategies[PARALLEL_STRATEGY] == null) {
+                final ZipStrategy zipStrategy = new ParallelZipStrategy(opts.getCompressThreads(),
+                    opts.getDeCompressThreads());
+                ZipStrategyManager.addZipStrategy(ZipStrategyManager.PARALLEL_STRATEGY, zipStrategy);
+                DEFAULT_STRATEGY = PARALLEL_STRATEGY;
+            }
+        }
+    }
+
+    private ZipStrategyManager() {
+    }
+
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Attachable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Attachable.java
new file mode 100644
index 0000000..5ffad99
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Attachable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface Attachable {
+
+    T getAttachments();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteArray.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteArray.java
new file mode 100644
index 0000000..b392fa7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteArray.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.Serializable;
+import java.util.Arrays;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * Can use it as a map key,
+ *
+ * @author jiachun.fjc
+ */
+public final class ByteArray implements Comparable, Serializable {
+
+    private static final long serialVersionUID = 3030232535108421145L;
+
+    private final byte[]      bytes;
+    // Cache the hash code, default to 0
+    private int               hashCode;
+
+    public static ByteArray wrap(final byte[] bytes) {
+        return new ByteArray(bytes);
+    }
+
+    ByteArray(byte[] bytes) {
+        Requires.requireNonNull(bytes, "bytes");
+        this.bytes = bytes;
+        // Initialize hash code to 0
+        this.hashCode = 0;
+    }
+
+    public byte[] getBytes() {
+        return bytes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        ByteArray that = (ByteArray) o;
+        // We intentionally use the function to compute hashcode here
+        return hashCode() == that.hashCode() && Arrays.equals(bytes, that.bytes);
+    }
+
+    @Override
+    public int hashCode() {
+        if (hashCode == 0) { // Lazy initialize
+            hashCode = Arrays.hashCode(bytes);
+        }
+        return hashCode;
+    }
+
+    @Override
+    public int compareTo(ByteArray o) {
+        return BytesUtil.compare(this.bytes, o.bytes);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectHashMap.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectHashMap.java
new file mode 100644
index 0000000..ec9d128
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectHashMap.java
@@ -0,0 +1,728 @@
+/*
+ * Copyright 2014 The Netty Project
+ *
+ * The Netty Project licenses this file to you under the Apache License, version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.AbstractCollection;
+import java.util.AbstractSet;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import com.alipay.sofa.jraft.util.Ints;
+
+/**
+ * A hash map implementation of {@link ByteObjectMap} that uses open addressing for keys.
+ * To minimize the memory footprint, this class uses open addressing rather than chaining.
+ * Collisions are resolved using linear probing. Deletions implement compaction, so cost of
+ * remove can approach O(N) for full maps, which makes a small loadFactor recommended.
+ *
+ * @param  The value type stored in the map.
+ */
+public class ByteObjectHashMap implements ByteObjectMap {
+
+    /**
+     * Default initial capacity. Used if not specified in the constructor
+     */
+    public static final int                   DEFAULT_CAPACITY    = 8;
+
+    /**
+     * Default load factor. Used if not specified in the constructor
+     */
+    public static final float                 DEFAULT_LOAD_FACTOR = 0.5f;
+
+    /**
+     * Placeholder for null values, so we can use the actual null to mean available.
+     * (Better than using a placeholder for available: less references for GC processing.)
+     */
+    private static final Object               NULL_VALUE          = new Object();
+
+    /**
+     * The maximum number of elements allowed without allocating more space.
+     */
+    private int                               maxSize;
+
+    /**
+     * The load factor for the map. Used to calculate {@link #maxSize}.
+     */
+    private final float                       loadFactor;
+
+    private byte[]                            keys;
+    private V[]                               values;
+    private int                               size;
+    private int                               mask;
+
+    private final Set                   keySet              = new KeySet();
+    private final Set>         entrySet            = new EntrySet();
+    private final Iterable> entries             = PrimitiveIterator::new;
+
+    public ByteObjectHashMap() {
+        this(DEFAULT_CAPACITY, DEFAULT_LOAD_FACTOR);
+    }
+
+    public ByteObjectHashMap(int initialCapacity) {
+        this(initialCapacity, DEFAULT_LOAD_FACTOR);
+    }
+
+    public ByteObjectHashMap(int initialCapacity, float loadFactor) {
+        if (initialCapacity < 1) {
+            throw new IllegalArgumentException("initialCapacity must be >= 1");
+        }
+        if (loadFactor <= 0.0f || loadFactor > 1.0f) {
+            // Cannot exceed 1 because we can never store more than capacity elements;
+            // using a bigger loadFactor would trigger rehashing before the desired load is reached.
+            throw new IllegalArgumentException("loadFactor must be > 0 and <= 1");
+        }
+
+        this.loadFactor = loadFactor;
+
+        // Adjust the initial capacity if necessary.
+        int capacity = Ints.findNextPositivePowerOfTwo(initialCapacity);
+        mask = capacity - 1;
+
+        // Allocate the arrays.
+        keys = new byte[capacity];
+        @SuppressWarnings({ "unchecked", "SuspiciousArrayCast" })
+        V[] temp = (V[]) new Object[capacity];
+        values = temp;
+
+        // Initialize the maximum size value.
+        maxSize = calcMaxSize(capacity);
+    }
+
+    private static  T toExternal(T value) {
+        return value == NULL_VALUE ? null : value;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static  T toInternal(T value) {
+        return value == null ? (T) NULL_VALUE : value;
+    }
+
+    @Override
+    public V get(byte key) {
+        int index = indexOf(key);
+        return index == -1 ? null : toExternal(values[index]);
+    }
+
+    @Override
+    public V put(byte key, V value) {
+        int startIndex = hashIndex(key);
+        int index = startIndex;
+
+        for (;;) {
+            if (values[index] == null) {
+                // Found empty slot, use it.
+                keys[index] = key;
+                values[index] = toInternal(value);
+                growSize();
+                return null;
+            }
+            if (keys[index] == key) {
+                // Found existing entry with this key, just replace the value.
+                V previousValue = values[index];
+                values[index] = toInternal(value);
+                return toExternal(previousValue);
+            }
+
+            // Conflict, keep probing ...
+            if ((index = probeNext(index)) == startIndex) {
+                // Can only happen if the map was full at MAX_ARRAY_SIZE and couldn't grow.
+                throw new IllegalStateException("Unable to insert");
+            }
+        }
+    }
+
+    @Override
+    public void putAll(Map sourceMap) {
+        if (sourceMap instanceof ByteObjectHashMap) {
+            // Optimization - iterate through the arrays.
+            @SuppressWarnings("unchecked")
+            ByteObjectHashMap source = (ByteObjectHashMap) sourceMap;
+            for (int i = 0; i < source.values.length; ++i) {
+                V sourceValue = source.values[i];
+                if (sourceValue != null) {
+                    put(source.keys[i], sourceValue);
+                }
+            }
+            return;
+        }
+
+        // Otherwise, just add each entry.
+        for (Entry entry : sourceMap.entrySet()) {
+            put(entry.getKey(), entry.getValue());
+        }
+    }
+
+    @Override
+    public V remove(byte key) {
+        int index = indexOf(key);
+        if (index == -1) {
+            return null;
+        }
+
+        V prev = values[index];
+        removeAt(index);
+        return toExternal(prev);
+    }
+
+    @Override
+    public int size() {
+        return size;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return size == 0;
+    }
+
+    @Override
+    public void clear() {
+        Arrays.fill(keys, (byte) 0);
+        Arrays.fill(values, null);
+        size = 0;
+    }
+
+    @Override
+    public boolean containsKey(byte key) {
+        return indexOf(key) >= 0;
+    }
+
+    @Override
+    public boolean containsValue(Object value) {
+        @SuppressWarnings("unchecked")
+        V v1 = toInternal((V) value);
+        for (V v2 : values) {
+            // The map supports null values; this will be matched as NULL_VALUE.equals(NULL_VALUE).
+            if (v2 != null && v2.equals(v1)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public Iterable> entries() {
+        return entries;
+    }
+
+    @Override
+    public Collection values() {
+        return new AbstractCollection() {
+            @Override
+            public Iterator iterator() {
+                return new Iterator() {
+                    final PrimitiveIterator iter = new PrimitiveIterator();
+
+                    @Override
+                    public boolean hasNext() {
+                        return iter.hasNext();
+                    }
+
+                    @Override
+                    public V next() {
+                        return iter.next().value();
+                    }
+
+                    @Override
+                    public void remove() {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            }
+
+            @Override
+            public int size() {
+                return size;
+            }
+        };
+    }
+
+    @Override
+    public int hashCode() {
+        // Hashcode is based on all non-zero, valid keys. We have to scan the whole keys
+        // array, which may have different lengths for two maps of same size(), so the
+        // capacity cannot be used as input for hashing but the size can.
+        int hash = size;
+        for (byte key : keys) {
+            // 0 can be a valid key or unused slot, but won't impact the hashcode in either case.
+            // This way we can use a cheap loop without conditionals, or hard-to-unroll operations,
+            // or the devastatingly bad memory locality of visiting value objects.
+            // Also, it's important to use a hash function that does not depend on the ordering
+            // of terms, only their values; since the map is an unordered collection and
+            // entries can end up in different positions in different maps that have the same
+            // elements, but with different history of puts/removes, due to conflicts.
+            hash ^= hashCode(key);
+        }
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ByteObjectMap)) {
+            return false;
+        }
+        @SuppressWarnings("rawtypes")
+        ByteObjectMap other = (ByteObjectMap) obj;
+        if (size != other.size()) {
+            return false;
+        }
+        for (int i = 0; i < values.length; ++i) {
+            V value = values[i];
+            if (value != null) {
+                byte key = keys[i];
+                Object otherValue = other.get(key);
+                if (value == NULL_VALUE) {
+                    if (otherValue != null) {
+                        return false;
+                    }
+                } else if (!value.equals(otherValue)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    @Override
+    public boolean containsKey(Object key) {
+        return containsKey(objectToKey(key));
+    }
+
+    @Override
+    public V get(Object key) {
+        return get(objectToKey(key));
+    }
+
+    @Override
+    public V put(Byte key, V value) {
+        return put(objectToKey(key), value);
+    }
+
+    @Override
+    public V remove(Object key) {
+        return remove(objectToKey(key));
+    }
+
+    @Override
+    public Set keySet() {
+        return keySet;
+    }
+
+    @Override
+    public Set> entrySet() {
+        return entrySet;
+    }
+
+    private byte objectToKey(Object key) {
+        return (Byte) key;
+    }
+
+    /**
+     * Locates the index for the given key. This method probes using double hashing.
+     *
+     * @param key the key for an entry in the map.
+     * @return the index where the key was found, or {@code -1} if no entry is found for that key.
+     */
+    private int indexOf(byte key) {
+        int startIndex = hashIndex(key);
+        int index = startIndex;
+
+        for (;;) {
+            if (values[index] == null) {
+                // It's available, so no chance that this value exists anywhere in the map.
+                return -1;
+            }
+            if (key == keys[index]) {
+                return index;
+            }
+
+            // Conflict, keep probing ...
+            if ((index = probeNext(index)) == startIndex) {
+                return -1;
+            }
+        }
+    }
+
+    /**
+     * Returns the hashed index for the given key.
+     */
+    private int hashIndex(byte key) {
+        // The array lengths are always a power of two, so we can use a bitmask to stay inside the array bounds.
+        return hashCode(key) & mask;
+    }
+
+    /**
+     * Returns the hash code for the key.
+     */
+    private static int hashCode(byte key) {
+        return (int) key;
+    }
+
+    /**
+     * Get the next sequential index after {@code index} and wraps if necessary.
+     */
+    private int probeNext(int index) {
+        // The array lengths are always a power of two, so we can use a bitmask to stay inside the array bounds.
+        return (index + 1) & mask;
+    }
+
+    /**
+     * Grows the map size after an insertion. If necessary, performs a rehash of the map.
+     */
+    private void growSize() {
+        size++;
+
+        if (size > maxSize) {
+            if (keys.length == Integer.MAX_VALUE) {
+                throw new IllegalStateException("Max capacity reached at size=" + size);
+            }
+
+            // Double the capacity.
+            rehash(keys.length << 1);
+        }
+    }
+
+    /**
+     * Removes entry at the given index position. Also performs opportunistic, incremental rehashing
+     * if necessary to not break conflict chains.
+     *
+     * @param index the index position of the element to remove.
+     * @return {@code true} if the next item was moved back. {@code false} otherwise.
+     */
+    private boolean removeAt(final int index) {
+        --size;
+        // Clearing the key is not strictly necessary (for GC like in a regular collection),
+        // but recommended for security. The memory location is still fresh in the cache anyway.
+        keys[index] = 0;
+        values[index] = null;
+
+        // In the interval from index to the next available entry, the arrays may have entries
+        // that are displaced from their base position due to prior conflicts. Iterate these
+        // entries and move them back if possible, optimizing future lookups.
+        // Knuth Section 6.4 Algorithm R, also used by the JDK's IdentityHashMap.
+
+        boolean movedBack = false;
+        int nextFree = index;
+        for (int i = probeNext(index); values[i] != null; i = probeNext(i)) {
+            int bucket = hashIndex(keys[i]);
+            if (i < bucket && (bucket <= nextFree || nextFree <= i) || bucket <= nextFree && nextFree <= i) {
+                // Move the displaced entry "back" to the first available position.
+                keys[nextFree] = keys[i];
+                values[nextFree] = values[i];
+                movedBack = true;
+                // Put the first entry after the displaced entry
+                keys[i] = 0;
+                values[i] = null;
+                nextFree = i;
+            }
+        }
+        return movedBack;
+    }
+
+    /**
+     * Calculates the maximum size allowed before rehashing.
+     */
+    private int calcMaxSize(int capacity) {
+        // Clip the upper bound so that there will always be at least one available slot.
+        int upperBound = capacity - 1;
+        return Math.min(upperBound, (int) (capacity * loadFactor));
+    }
+
+    /**
+     * Rehashes the map for the given capacity.
+     *
+     * @param newCapacity the new capacity for the map.
+     */
+    private void rehash(int newCapacity) {
+        byte[] oldKeys = keys;
+        V[] oldVals = values;
+
+        keys = new byte[newCapacity];
+        @SuppressWarnings({ "unchecked", "SuspiciousArrayCast" })
+        V[] temp = (V[]) new Object[newCapacity];
+        values = temp;
+
+        maxSize = calcMaxSize(newCapacity);
+        mask = newCapacity - 1;
+
+        // Insert to the new arrays.
+        for (int i = 0; i < oldVals.length; ++i) {
+            V oldVal = oldVals[i];
+            if (oldVal != null) {
+                // Inlined put(), but much simpler: we don't need to worry about
+                // duplicated keys, growing/rehashing, or failing to insert.
+                byte oldKey = oldKeys[i];
+                int index = hashIndex(oldKey);
+
+                for (;;) {
+                    if (values[index] == null) {
+                        keys[index] = oldKey;
+                        values[index] = oldVal;
+                        break;
+                    }
+
+                    // Conflict, keep probing. Can wrap around, but never reaches startIndex again.
+                    index = probeNext(index);
+                }
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        if (isEmpty()) {
+            return "{}";
+        }
+        StringBuilder sb = new StringBuilder(4 * size);
+        sb.append('{');
+        boolean first = true;
+        for (int i = 0; i < values.length; ++i) {
+            V value = values[i];
+            if (value != null) {
+                if (!first) {
+                    sb.append(", ");
+                }
+                sb.append(keyToString(keys[i])).append('=').append(value == this ? "(this Map)" : toExternal(value));
+                first = false;
+            }
+        }
+        return sb.append('}').toString();
+    }
+
+    /**
+     * Helper method called by {@link #toString()} in order to convert a single map key into a string.
+     * This is protected to allow subclasses to override the appearance of a given key.
+     */
+    protected String keyToString(byte key) {
+        return Byte.toString(key);
+    }
+
+    /**
+     * Set implementation for iterating over the entries of the map.
+     */
+    private final class EntrySet extends AbstractSet> {
+        @Override
+        public Iterator> iterator() {
+            return new MapIterator();
+        }
+
+        @Override
+        public int size() {
+            return ByteObjectHashMap.this.size();
+        }
+    }
+
+    /**
+     * Set implementation for iterating over the keys.
+     */
+    private final class KeySet extends AbstractSet {
+        @Override
+        public int size() {
+            return ByteObjectHashMap.this.size();
+        }
+
+        @Override
+        public boolean contains(Object o) {
+            return ByteObjectHashMap.this.containsKey(o);
+        }
+
+        @Override
+        public boolean remove(Object o) {
+            return ByteObjectHashMap.this.remove(o) != null;
+        }
+
+        @Override
+        public boolean retainAll(Collection retainedKeys) {
+            boolean changed = false;
+            for (Iterator> iter = entries().iterator(); iter.hasNext();) {
+                PrimitiveEntry entry = iter.next();
+                if (!retainedKeys.contains(entry.key())) {
+                    changed = true;
+                    iter.remove();
+                }
+            }
+            return changed;
+        }
+
+        @Override
+        public void clear() {
+            ByteObjectHashMap.this.clear();
+        }
+
+        @Override
+        public Iterator iterator() {
+            return new Iterator() {
+                private final Iterator> iter = entrySet.iterator();
+
+                @Override
+                public boolean hasNext() {
+                    return iter.hasNext();
+                }
+
+                @Override
+                public Byte next() {
+                    return iter.next().getKey();
+                }
+
+                @Override
+                public void remove() {
+                    iter.remove();
+                }
+            };
+        }
+    }
+
+    /**
+     * Iterator over primitive entries. Entry key/values are overwritten by each call to {@link #next()}.
+     */
+    private final class PrimitiveIterator implements Iterator>, PrimitiveEntry {
+        private int prevIndex  = -1;
+        private int nextIndex  = -1;
+        private int entryIndex = -1;
+
+        private void scanNext() {
+            for (;;) {
+                if (++nextIndex == values.length || values[nextIndex] != null) {
+                    break;
+                }
+            }
+        }
+
+        @Override
+        public boolean hasNext() {
+            if (nextIndex == -1) {
+                scanNext();
+            }
+            return nextIndex < keys.length;
+        }
+
+        @Override
+        public PrimitiveEntry next() {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+
+            prevIndex = nextIndex;
+            scanNext();
+
+            // Always return the same Entry object, just change its index each time.
+            entryIndex = prevIndex;
+            return this;
+        }
+
+        @Override
+        public void remove() {
+            if (prevIndex < 0) {
+                throw new IllegalStateException("next must be called before each remove.");
+            }
+            if (removeAt(prevIndex)) {
+                // removeAt may move elements "back" in the array if they have been displaced because their spot in the
+                // array was occupied when they were inserted. If this occurs then the nextIndex is now invalid and
+                // should instead point to the prevIndex which now holds an element which was "moved back".
+                nextIndex = prevIndex;
+            }
+            prevIndex = -1;
+        }
+
+        // Entry implementation. Since this implementation uses a single Entry, we coalesce that
+        // into the Iterator object (potentially making loop optimization much easier).
+
+        @Override
+        public byte key() {
+            return keys[entryIndex];
+        }
+
+        @Override
+        public V value() {
+            return toExternal(values[entryIndex]);
+        }
+
+        @Override
+        public void setValue(V value) {
+            values[entryIndex] = toInternal(value);
+        }
+    }
+
+    /**
+     * Iterator used by the {@link Map} interface.
+     */
+    private final class MapIterator implements Iterator> {
+        private final PrimitiveIterator iter = new PrimitiveIterator();
+
+        @Override
+        public boolean hasNext() {
+            return iter.hasNext();
+        }
+
+        @Override
+        public Entry next() {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+
+            iter.next();
+
+            return new MapEntry(iter.entryIndex);
+        }
+
+        @Override
+        public void remove() {
+            iter.remove();
+        }
+    }
+
+    /**
+     * A single entry in the map.
+     */
+    final class MapEntry implements Entry {
+        private final int entryIndex;
+
+        MapEntry(int entryIndex) {
+            this.entryIndex = entryIndex;
+        }
+
+        @Override
+        public Byte getKey() {
+            verifyExists();
+            return keys[entryIndex];
+        }
+
+        @Override
+        public V getValue() {
+            verifyExists();
+            return toExternal(values[entryIndex]);
+        }
+
+        @Override
+        public V setValue(V value) {
+            verifyExists();
+            V prevValue = toExternal(values[entryIndex]);
+            values[entryIndex] = toInternal(value);
+            return prevValue;
+        }
+
+        private void verifyExists() {
+            if (values[entryIndex] == null) {
+                throw new IllegalStateException("The map entry has been removed");
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectMap.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectMap.java
new file mode 100644
index 0000000..5289efb
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ByteObjectMap.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.Map;
+
+/**
+ * Interface for a primitive map that uses {@code byte}s as keys.
+ *
+ * @param  the value type stored in the map.
+ */
+public interface ByteObjectMap extends Map {
+
+    /**
+     * A primitive entry in the map, provided by the iterator from {@link #entries()}
+     *
+     * @param  the value type stored in the map.
+     */
+    interface PrimitiveEntry {
+        /**
+         * Gets the key for this entry.
+         */
+        byte key();
+
+        /**
+         * Gets the value for this entry.
+         */
+        V value();
+
+        /**
+         * Sets the value for this entry.
+         */
+        void setValue(V value);
+    }
+
+    /**
+     * Gets the value in the map with the specified key.
+     *
+     * @param key the key whose associated value is to be returned.
+     * @return the value or {@code null} if the key was not found in the map.
+     */
+    V get(byte key);
+
+    /**
+     * Puts the given entry into the map.
+     *
+     * @param key   the key of the entry.
+     * @param value the value of the entry.
+     * @return the previous value for this key or {@code null} if there was no previous mapping.
+     */
+    V put(byte key, V value);
+
+    /**
+     * Removes the entry with the specified key.
+     *
+     * @param key the key for the entry to be removed from this map.
+     * @return the previous value for the key, or {@code null} if there was no mapping.
+     */
+    V remove(byte key);
+
+    /**
+     * Gets an iterable to traverse over the primitive entries contained in this map. As an optimization,
+     * the {@link PrimitiveEntry}s returned by the Iterator may change as the Iterator
+     * progresses. The caller should not rely on {@link PrimitiveEntry} key/value stability.
+     */
+    Iterable> entries();
+
+    /**
+     * Indicates whether or not this map contains a value for the specified key.
+     */
+    boolean containsKey(byte key);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Clock.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Clock.java
new file mode 100644
index 0000000..a125bdc
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Clock.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+/**
+ * An abstraction for how time passes.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class Clock {
+
+    /**
+     * Returns the current time tick.
+     *
+     * @return time tick in nanoseconds
+     */
+    public abstract long getTick();
+
+    /**
+     * Returns the current time in milliseconds.
+     *
+     * @return time in milliseconds
+     */
+    public long getTime() {
+        return System.currentTimeMillis();
+    }
+
+    /**
+     * The default clock to use.
+     *
+     * @return the default {@link Clock} instance
+     * @see Clock.UserTimeClock
+     */
+    public static Clock defaultClock() {
+        return UserTimeClockHolder.DEFAULT;
+    }
+
+    /**
+     * A clock implementation which returns the current time in epoch nanoseconds.
+     */
+    public static class UserTimeClock extends Clock {
+
+        @Override
+        public long getTick() {
+            return System.nanoTime();
+        }
+    }
+
+    private static class UserTimeClockHolder {
+
+        private static final Clock DEFAULT = new UserTimeClock();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Configured.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Configured.java
new file mode 100644
index 0000000..bd7a4dd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Configured.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface Configured {
+
+    T config();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Constants.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Constants.java
new file mode 100644
index 0000000..2ef624f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Constants.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.Formatter;
+
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+
+/**
+ * Rhea's constants
+ *
+ * @author jiachun.fjc
+ */
+public final class Constants {
+
+    public static final String  NEWLINE;
+
+    static {
+        String newLine;
+        try {
+            newLine = new Formatter().format("%n").toString();
+        } catch (final Exception e) {
+            newLine = "\n";
+        }
+        NEWLINE = newLine;
+    }
+
+    public static final boolean THREAD_AFFINITY_ENABLED = SystemPropertyUtil.getBoolean("rhea.thread.affinity.enabled",
+                                                            false);
+
+    public static final long    DEFAULT_REGION_ID       = -1L;
+
+    private Constants() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/JvmTools.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/JvmTools.java
new file mode 100644
index 0000000..d7735c5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/JvmTools.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.File;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryMXBean;
+import java.lang.management.MemoryUsage;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import com.sun.management.HotSpotDiagnosticMXBean;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class JvmTools {
+
+    /**
+     * Returns java stack traces of java threads for the current
+     * java process.
+     */
+    public static List jStack() throws Exception {
+        final List stackList = new LinkedList<>();
+        final Map allStackTraces = Thread.getAllStackTraces();
+        for (final Map.Entry entry : allStackTraces.entrySet()) {
+            final Thread thread = entry.getKey();
+            final StackTraceElement[] stackTraces = entry.getValue();
+
+            stackList.add(String.format("\"%s\" tid=%s isDaemon=%s priority=%s" + Constants.NEWLINE, thread.getName(),
+                thread.getId(), thread.isDaemon(), thread.getPriority()));
+
+            stackList.add("java.lang.Thread.State: " + thread.getState() + Constants.NEWLINE);
+
+            if (stackTraces != null) {
+                for (final StackTraceElement s : stackTraces) {
+                    stackList.add("    " + s.toString() + Constants.NEWLINE);
+                }
+            }
+        }
+        return stackList;
+    }
+
+    /**
+     * Returns memory usage for the current java process.
+     */
+    public static List memoryUsage() throws Exception {
+        final MemoryUsage heapMemoryUsage = MXBeanHolder.memoryMxBean.getHeapMemoryUsage();
+        final MemoryUsage nonHeapMemoryUsage = MXBeanHolder.memoryMxBean.getNonHeapMemoryUsage();
+
+        final List memoryUsageList = new LinkedList<>();
+        memoryUsageList.add("********************************** Memory Usage **********************************"
+                            + Constants.NEWLINE);
+        memoryUsageList.add("Heap Memory Usage: " + heapMemoryUsage.toString() + Constants.NEWLINE);
+        memoryUsageList.add("NonHeap Memory Usage: " + nonHeapMemoryUsage.toString() + Constants.NEWLINE);
+
+        return memoryUsageList;
+    }
+
+    /**
+     * Returns the heap memory used for the current java process.
+     */
+    public static double memoryUsed() throws Exception {
+        final MemoryUsage heapMemoryUsage = MXBeanHolder.memoryMxBean.getHeapMemoryUsage();
+        return (double) (heapMemoryUsage.getUsed()) / heapMemoryUsage.getMax();
+    }
+
+    /**
+     * Dumps the heap to the outputFile file in the same format as the
+     * hprof heap dump.
+     * @param outputFile    the system-dependent filename
+     * @param live          if true dump only live objects i.e. objects
+     *                      that are reachable from others
+     */
+    @SuppressWarnings("all")
+    public static void jMap(final String outputFile, final boolean live) throws Exception {
+        final File file = new File(outputFile);
+        if (file.exists()) {
+            file.delete();
+        }
+        MXBeanHolder.hotSpotDiagnosticMxBean.dumpHeap(outputFile, live);
+    }
+
+    private static class MXBeanHolder {
+        static final MemoryMXBean            memoryMxBean            = ManagementFactory.getMemoryMXBean();
+        static final HotSpotDiagnosticMXBean hotSpotDiagnosticMxBean = ManagementFactory
+                                                                         .getPlatformMXBean(HotSpotDiagnosticMXBean.class);
+    }
+
+    private JvmTools() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/KVParameterRequires.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/KVParameterRequires.java
new file mode 100644
index 0000000..2b92fa1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/KVParameterRequires.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.cmd.store.BaseRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.errors.InvalidParameterException;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class KVParameterRequires {
+
+    public static void requireSameEpoch(final BaseRequest request, final RegionEpoch current) {
+        RegionEpoch requestEpoch = request.getRegionEpoch();
+        if (current.equals(requestEpoch)) {
+            return;
+        }
+        if (current.getConfVer() != requestEpoch.getConfVer()) {
+            throw Errors.INVALID_REGION_MEMBERSHIP.exception();
+        }
+        if (current.getVersion() != requestEpoch.getVersion()) {
+            throw Errors.INVALID_REGION_VERSION.exception();
+        }
+        throw Errors.INVALID_REGION_EPOCH.exception();
+    }
+
+    public static  T requireNonNull(final T target, final String message) {
+        if (target == null) {
+            throw new InvalidParameterException(message);
+        }
+        return target;
+    }
+
+    public static  List requireNonEmpty(final List target, final String message) {
+        requireNonNull(target, message);
+        if (target.isEmpty()) {
+            throw new InvalidParameterException(message);
+        }
+        return target;
+    }
+
+    public static int requireNonNegative(final int value, final String message) {
+        if (value < 0) {
+            throw new InvalidParameterException(message);
+        }
+        return value;
+    }
+
+    public static long requireNonNegative(final long value, final String message) {
+        if (value < 0) {
+            throw new InvalidParameterException(message);
+        }
+        return value;
+    }
+
+    public static int requirePositive(final int value, final String message) {
+        if (value <= 0) {
+            throw new InvalidParameterException(message);
+        }
+        return value;
+    }
+
+    public static long requirePositive(final long value, final String message) {
+        if (value <= 0) {
+            throw new InvalidParameterException(message);
+        }
+        return value;
+    }
+
+    private KVParameterRequires() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Lists.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Lists.java
new file mode 100644
index 0000000..2d195a9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Lists.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.Serializable;
+import java.util.AbstractList;
+import java.util.AbstractSequentialList;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.RandomAccess;
+import java.util.function.Function;
+
+import com.alipay.sofa.jraft.util.Ints;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * Static utility methods pertaining to {@link List} instances.
+ *
+ * @author jiachun.fjc
+ */
+public final class Lists {
+
+    /**
+     * Creates a mutable, empty {@code ArrayList} instance.
+     */
+    public static  ArrayList newArrayList() {
+        return new ArrayList<>();
+    }
+
+    /**
+     * Creates a mutable {@code ArrayList} instance containing the given elements.
+     */
+    @SuppressWarnings("unchecked")
+    public static  ArrayList newArrayList(final E... elements) {
+        Requires.requireNonNull(elements, "elements");
+        // Avoid integer overflow when a large array is passed in
+        final int capacity = computeArrayListCapacity(elements.length);
+        final ArrayList list = new ArrayList<>(capacity);
+        Collections.addAll(list, elements);
+        return list;
+    }
+
+    /**
+     * Creates a mutable {@code ArrayList} instance containing the given elements.
+     */
+    @SuppressWarnings("unchecked")
+    public static  ArrayList newArrayList(final Iterable elements) {
+        Requires.requireNonNull(elements, "elements");
+        return elements instanceof Collection ? new ArrayList((Collection) elements) : newArrayList(elements
+            .iterator());
+    }
+
+    /**
+     * Creates a mutable {@code ArrayList} instance containing the given elements.
+     */
+    public static  ArrayList newArrayList(final Iterator elements) {
+        final ArrayList list = newArrayList();
+        while (elements.hasNext()) {
+            list.add(elements.next());
+        }
+        return list;
+    }
+
+    /**
+     * Creates an {@code ArrayList} instance backed by an array of the exact size specified;
+     * equivalent to {@link ArrayList#ArrayList(int)}.
+     */
+    public static  ArrayList newArrayListWithCapacity(final int initialArraySize) {
+        Requires.requireTrue(initialArraySize >= 0, "initialArraySize");
+        return new ArrayList<>(initialArraySize);
+    }
+
+    /**
+     * Returns a list that applies {@code function} to each element of {@code fromList}.
+     * The returned list is a transformed view of {@code fromList};
+     * changes to {@code fromList} will be reflected in the returned list and vice versa.
+     */
+    public static  List transform(final List fromList, final Function function) {
+        return (fromList instanceof RandomAccess) ? new TransformingRandomAccessList<>(fromList, function)
+            : new TransformingSequentialList<>(fromList, function);
+    }
+
+    private static class TransformingRandomAccessList extends AbstractList implements RandomAccess,
+                                                                                   Serializable {
+        private static final long              serialVersionUID = 0;
+
+        final List                          fromList;
+        final Function function;
+
+        TransformingRandomAccessList(List fromList, Function function) {
+            this.fromList = Requires.requireNonNull(fromList, "fromList");
+            this.function = Requires.requireNonNull(function, "function");
+        }
+
+        @Override
+        public void clear() {
+            fromList.clear();
+        }
+
+        @Override
+        public T get(final int index) {
+            return function.apply(fromList.get(index));
+        }
+
+        @Override
+        public boolean isEmpty() {
+            return fromList.isEmpty();
+        }
+
+        @Override
+        public T remove(final int index) {
+            return function.apply(fromList.remove(index));
+        }
+
+        @Override
+        public int size() {
+            return fromList.size();
+        }
+    }
+
+    private static class TransformingSequentialList extends AbstractSequentialList implements Serializable {
+        private static final long              serialVersionUID = 0;
+
+        final List                          fromList;
+        final Function function;
+
+        TransformingSequentialList(List fromList, Function function) {
+            this.fromList = Requires.requireNonNull(fromList, "fromList");
+            this.function = Requires.requireNonNull(function, "function");
+        }
+
+        @Override
+        public void clear() {
+            fromList.clear();
+        }
+
+        @Override
+        public int size() {
+            return fromList.size();
+        }
+
+        @Override
+        public ListIterator listIterator(final int index) {
+            return new TransformedListIterator(fromList.listIterator(index)) {
+
+                @Override
+                T transform(F from) {
+                    return function.apply(from);
+                }
+            };
+        }
+    }
+
+    abstract static class TransformedIterator implements Iterator {
+
+        final Iterator backingIterator;
+
+        TransformedIterator(Iterator backingIterator) {
+            this.backingIterator = Requires.requireNonNull(backingIterator, "backingIterator");
+        }
+
+        abstract T transform(final F from);
+
+        @Override
+        public final boolean hasNext() {
+            return backingIterator.hasNext();
+        }
+
+        @Override
+        public final T next() {
+            return transform(backingIterator.next());
+        }
+
+        @Override
+        public final void remove() {
+            backingIterator.remove();
+        }
+    }
+
+    abstract static class TransformedListIterator extends TransformedIterator implements ListIterator {
+
+        TransformedListIterator(ListIterator backingIterator) {
+            super(backingIterator);
+        }
+
+        @SuppressWarnings("unchecked")
+        private ListIterator backingIterator() {
+            return (ListIterator) backingIterator;
+        }
+
+        @Override
+        public final boolean hasPrevious() {
+            return backingIterator().hasPrevious();
+        }
+
+        @Override
+        public final T previous() {
+            return transform(backingIterator().previous());
+        }
+
+        @Override
+        public final int nextIndex() {
+            return backingIterator().nextIndex();
+        }
+
+        @Override
+        public final int previousIndex() {
+            return backingIterator().previousIndex();
+        }
+
+        @Override
+        public void set(final T element) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void add(final T element) {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    static int computeArrayListCapacity(final int arraySize) {
+        Requires.requireTrue(arraySize >= 0, "arraySize");
+        return Ints.saturatedCast(5L + arraySize + (arraySize / 10));
+    }
+
+    private Lists() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Maps.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Maps.java
new file mode 100644
index 0000000..5760f59
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Maps.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.HashMap;
+import java.util.IdentityHashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.collection.NonBlockingHashMap;
+import com.alipay.sofa.jraft.rhea.util.concurrent.collection.NonBlockingHashMapLong;
+import com.alipay.sofa.jraft.util.Ints;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * Static utility methods pertaining to {@link Map} instances.
+ *
+ * @author jiachun.fjc
+ */
+public final class Maps {
+
+    private static final boolean USE_NON_BLOCKING_HASH = SystemPropertyUtil.getBoolean("rhea.use.non_blocking_hash",
+                                                           true);
+
+    /**
+     * Creates a mutable, empty {@code HashMap} instance.
+     */
+    public static  HashMap newHashMap() {
+        return new HashMap<>();
+    }
+
+    /**
+     * Creates a {@code HashMap} instance, with a high enough "initial capacity"
+     * that it should hold {@code expectedSize} elements without growth.
+     */
+    public static  HashMap newHashMapWithExpectedSize(int expectedSize) {
+        return new HashMap<>(capacity(expectedSize));
+    }
+
+    /**
+     * Creates an {@code IdentityHashMap} instance.
+     */
+    public static  IdentityHashMap newIdentityHashMap() {
+        return new IdentityHashMap<>();
+    }
+
+    /**
+     * Creates an {@code IdentityHashMap} instance, with a high enough "initial capacity"
+     * that it should hold {@code expectedSize} elements without growth.
+     */
+    public static  IdentityHashMap newIdentityHashMapWithExpectedSize(int expectedSize) {
+        return new IdentityHashMap<>(capacity(expectedSize));
+    }
+
+    /**
+     * Creates a mutable, empty, insertion-ordered {@code LinkedHashMap} instance.
+     */
+    public static  LinkedHashMap newLinkedHashMap() {
+        return new LinkedHashMap<>();
+    }
+
+    /**
+     * Creates a mutable, empty {@code TreeMap} instance using the natural ordering of its elements.
+     */
+    public static  TreeMap newTreeMap() {
+        return new TreeMap<>();
+    }
+
+    /**
+     * Creates a mutable, empty {@code ConcurrentMap} instance.
+     */
+    public static  ConcurrentMap newConcurrentMap() {
+        if (USE_NON_BLOCKING_HASH && UnsafeUtil.hasUnsafe()) {
+            return new NonBlockingHashMap<>();
+        }
+        return new ConcurrentHashMap<>();
+    }
+
+    /**
+     * Creates a {@code ConcurrentMap} instance, with a high enough "initial capacity"
+     * that it should hold {@code expectedSize} elements without growth.
+     */
+    public static  ConcurrentMap newConcurrentMap(int initialCapacity) {
+        if (USE_NON_BLOCKING_HASH && UnsafeUtil.hasUnsafe()) {
+            return new NonBlockingHashMap<>(initialCapacity);
+        }
+        return new ConcurrentHashMap<>(initialCapacity);
+    }
+
+    /**
+     * Creates a mutable, empty {@code NonBlockingHashMapLong} instance.
+     */
+    public static  ConcurrentMap newConcurrentMapLong() {
+        if (USE_NON_BLOCKING_HASH && UnsafeUtil.hasUnsafe()) {
+            return new NonBlockingHashMapLong<>();
+        }
+        return new ConcurrentHashMap<>();
+    }
+
+    /**
+     * Creates a {@code NonBlockingHashMapLong} instance, with a high enough "initial capacity"
+     * that it should hold {@code expectedSize} elements without growth.
+     */
+    public static  ConcurrentMap newConcurrentMapLong(int initialCapacity) {
+        if (USE_NON_BLOCKING_HASH) {
+            return new NonBlockingHashMapLong<>(initialCapacity);
+        }
+        return new ConcurrentHashMap<>(initialCapacity);
+    }
+
+    /**
+     * Returns a capacity that is sufficient to keep the map from being resized as
+     * long as it grows no larger than expectedSize and the load factor is >= its
+     * default (0.75).
+     */
+    private static int capacity(int expectedSize) {
+        if (expectedSize < 3) {
+            Requires.requireTrue(expectedSize >= 0, "expectedSize cannot be negative but was: " + expectedSize);
+            return expectedSize + 1;
+        }
+        if (expectedSize < Ints.MAX_POWER_OF_TWO) {
+            return expectedSize + expectedSize / 3;
+        }
+        return Integer.MAX_VALUE; // any large value
+    }
+
+    private Maps() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/NetUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/NetUtil.java
new file mode 100644
index 0000000..88f2c8b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/NetUtil.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+import java.util.regex.Pattern;
+
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class NetUtil {
+
+    private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3}$");
+    private static final String  LOCAL_IP_ADDRESS;
+
+    static {
+        InetAddress localAddress;
+        try {
+            localAddress = InetAddress.getLocalHost();
+        } catch (UnknownHostException e) {
+            localAddress = null;
+        }
+
+        if (localAddress != null && isValidAddress(localAddress)) {
+            LOCAL_IP_ADDRESS = localAddress.getHostAddress();
+        } else {
+            LOCAL_IP_ADDRESS = getFirstLocalAddress();
+        }
+    }
+
+    public static String getLocalAddress() {
+        return LOCAL_IP_ADDRESS;
+    }
+
+    public static String getLocalHostName() {
+        try {
+            return InetAddress.getLocalHost().getHostName();
+        } catch (final UnknownHostException e) {
+            ThrowUtil.throwException(e);
+        }
+        return null; // never get here
+    }
+
+    /**
+     * Gets the fully qualified domain name for this IP address.
+     * Best effort method, meaning we may not be able to return
+     * the FQDN depending on the underlying system configuration.
+     *
+     * @return the fully qualified domain name for this IP address,
+     * or if the operation is not allowed by the security check,
+     * the textual representation of the IP address.
+     */
+    public static String getLocalCanonicalHostName() {
+        try {
+            return InetAddress.getLocalHost().getCanonicalHostName();
+        } catch (final UnknownHostException ignored) {
+            return getLocalHostName();
+        }
+    }
+
+    /**
+     * Gets the first valid IP in the NIC
+     */
+    private static String getFirstLocalAddress() {
+        try {
+            final Enumeration interfaces = NetworkInterface.getNetworkInterfaces();
+            while (interfaces.hasMoreElements()) {
+                final NetworkInterface ni = interfaces.nextElement();
+                final Enumeration addresses = ni.getInetAddresses();
+                while (addresses.hasMoreElements()) {
+                    InetAddress address = addresses.nextElement();
+                    if (!address.isLoopbackAddress() && !address.getHostAddress().contains(":")) {
+                        return address.getHostAddress();
+                    }
+                }
+            }
+        } catch (final Throwable ignored) {
+            // ignored
+        }
+
+        return "127.0.0.1";
+    }
+
+    private static boolean isValidAddress(final InetAddress address) {
+        if (address.isLoopbackAddress()) {
+            return false;
+        }
+
+        final String name = address.getHostAddress();
+        return (name != null && !"0.0.0.0".equals(name) && !"127.0.0.1".equals(name) && IP_PATTERN.matcher(name)
+            .matches());
+    }
+
+    private NetUtil() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Pair.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Pair.java
new file mode 100644
index 0000000..28ef11b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Pair.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.Serializable;
+import java.util.Objects;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class Pair implements Serializable {
+
+    private static final long serialVersionUID = 3390134330336111160L;
+
+    private K                 key;
+    private V                 value;
+
+    public static  Pair of(K key, V value) {
+        return new Pair<>(key, value);
+    }
+
+    public Pair(K key, V value) {
+        this.key = key;
+        this.value = value;
+    }
+
+    public K getKey() {
+        return key;
+    }
+
+    public void setKey(K key) {
+        this.key = key;
+    }
+
+    public V getValue() {
+        return value;
+    }
+
+    public void setValue(V value) {
+        this.value = value;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        Pair pair = (Pair) o;
+        return Objects.equals(key, pair.key) && Objects.equals(value, pair.value);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(key, value);
+    }
+
+    @Override
+    public String toString() {
+        return "Pair{" + "key=" + key + ", value=" + value + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Partitions.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Partitions.java
new file mode 100644
index 0000000..389b874
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Partitions.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * Batch tasks are split and executed in the smallest unit,
+ * can be used to perform parallel tasks, and can also be
+ * used to perform serial batch tasks.
+ *
+ * @author jiachun.fjc
+ */
+public class Partitions {
+
+    /**
+     * Batch tasks are split and executed in the smallest unit,
+     * and the callback function (input -> output) is in a
+     * many-to-one manner.
+     *
+     * @param inputList input parameter list
+     * @param unitSize  the minimum unit
+     * @param func      the callback function
+     * @param       input type
+     * @param      output type
+     */
+    public static  List manyToOne(final List inputList, final int unitSize,
+                                                final Function, OUT> func) {
+        if (inputList == null || inputList.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        final int inputSize = inputList.size();
+
+        final List outputList = Lists.newArrayListWithCapacity((int) Math.ceil((double) inputSize / unitSize));
+
+        if (inputSize <= unitSize) {
+            add(outputList, func.apply(inputList));
+            return outputList;
+        }
+
+        List segment = Lists.newArrayListWithCapacity(unitSize);
+        for (final IN input : inputList) {
+            segment.add(input);
+            if (segment.size() >= unitSize) {
+                add(outputList, func.apply(segment));
+                segment = Lists.newArrayListWithCapacity(unitSize);
+            }
+        }
+
+        if (!segment.isEmpty()) {
+            add(outputList, func.apply(segment));
+        }
+
+        return outputList;
+    }
+
+    /**
+     * Batch tasks are split and executed in the smallest unit,
+     * and the callback function (input -> output) is in a
+     * one-to-one manner.
+     *
+     * @param inputList input parameter list
+     * @param unitSize  the minimum unit
+     * @param func      the callback function
+     * @param       input type
+     * @param      output type
+     */
+    public static  List oneToOne(final List inputList, final int unitSize,
+                                               final Function, List> func) {
+        if (inputList == null || inputList.isEmpty()) {
+            return Collections.emptyList();
+        }
+
+        final int inputSize = inputList.size();
+
+        if (inputSize <= unitSize) {
+            return func.apply(inputList);
+        }
+
+        final List outputList = Lists.newArrayListWithCapacity(inputSize);
+
+        List segment = Lists.newArrayListWithCapacity(unitSize);
+        for (final IN input : inputList) {
+            segment.add(input);
+            if (segment.size() >= unitSize) {
+                addAll(outputList, func.apply(segment));
+                segment = Lists.newArrayListWithCapacity(unitSize);
+            }
+        }
+
+        if (!segment.isEmpty()) {
+            addAll(outputList, func.apply(segment));
+        }
+
+        return outputList;
+    }
+
+    private static  void add(final List list, final T element) {
+        if (list != null && element != null) {
+            list.add(element);
+        }
+    }
+
+    private static  void addAll(final List list, final List elements) {
+        if (list != null && elements != null && !elements.isEmpty()) {
+            list.addAll(elements);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/RegionHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/RegionHelper.java
new file mode 100644
index 0000000..6aade9c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/RegionHelper.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class RegionHelper {
+
+    public static boolean isSingleGroup(final Region region) {
+        return BytesUtil.nullToEmpty(region.getStartKey()).length == 0
+               && BytesUtil.nullToEmpty(region.getEndKey()).length == 0;
+    }
+
+    public static boolean isMultiGroup(final Region region) {
+        return !isSingleGroup(region);
+    }
+
+    public static boolean isSameRange(final Region r1, final Region r2) {
+        if (BytesUtil.compare(BytesUtil.nullToEmpty(r1.getStartKey()), BytesUtil.nullToEmpty(r2.getStartKey())) != 0) {
+            return false;
+        }
+        return BytesUtil.compare(BytesUtil.nullToEmpty(r1.getEndKey()), BytesUtil.nullToEmpty(r2.getEndKey())) == 0;
+    }
+
+    public static boolean isKeyInRegion(final byte[] key, final Region region) {
+        final byte[] startKey = BytesUtil.nullToEmpty(region.getStartKey());
+        if (BytesUtil.compare(key, startKey) < 0) {
+            return false;
+        }
+        final byte[] endKey = region.getEndKey();
+        return endKey == null || BytesUtil.compare(key, endKey) < 0;
+    }
+
+    private RegionHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StackTraceUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StackTraceUtil.java
new file mode 100644
index 0000000..bb8794c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StackTraceUtil.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class StackTraceUtil {
+
+    private static final String NULL_STRING = "null";
+
+    public static String stackTrace(final Throwable t) {
+        if (t == null) {
+            return NULL_STRING;
+        }
+
+        try (final ByteArrayOutputStream out = new ByteArrayOutputStream(); final PrintStream ps = new PrintStream(out)) {
+            t.printStackTrace(ps);
+            ps.flush();
+            return new String(out.toByteArray());
+        } catch (final IOException ignored) {
+            // ignored
+        }
+        return NULL_STRING;
+    }
+
+    private StackTraceUtil() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StringBuilderHelper.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StringBuilderHelper.java
new file mode 100644
index 0000000..5bcbdd6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/StringBuilderHelper.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import com.alipay.sofa.jraft.util.internal.ReferenceFieldUpdater;
+import com.alipay.sofa.jraft.util.internal.Updaters;
+
+/**
+ * Reuse {@link StringBuilder} based on {@link ThreadLocal}.
+ *
+ * Be careful that do not to nest in the same thread.
+ *
+ * @author jiachun.fjc
+ */
+public class StringBuilderHelper {
+
+    private static final ReferenceFieldUpdater valueUpdater      = Updaters
+                                                                                            .newReferenceFieldUpdater(
+                                                                                                StringBuilder.class
+                                                                                                    .getSuperclass(),
+                                                                                                "value");
+
+    private static final int                                          DISCARD_LIMIT     = 1024 << 3;     // 8k
+
+    private static final ThreadLocal             holderThreadLocal = ThreadLocal
+                                                                                            .withInitial(StringBuilderHolder::new);
+
+    public static StringBuilder get() {
+        final StringBuilderHolder holder = holderThreadLocal.get();
+        return holder.getStringBuilder();
+    }
+
+    public static void truncate() {
+        final StringBuilderHolder holder = holderThreadLocal.get();
+        holder.truncate();
+    }
+
+    private static class StringBuilderHolder {
+
+        private final StringBuilder buf = new StringBuilder();
+
+        private StringBuilder getStringBuilder() {
+            truncate();
+            return buf;
+        }
+
+        private void truncate() {
+            if (buf.capacity() > DISCARD_LIMIT) {
+                valueUpdater.set(buf, new char[1024]);
+            }
+            buf.setLength(0);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Strings.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Strings.java
new file mode 100644
index 0000000..300a275
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/Strings.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.List;
+
+/**
+ * Static utility methods pertaining to {@code String} or {@code CharSequence}
+ * instances.
+ *
+ * @author jiachun.fjc
+ */
+public final class Strings {
+
+    /**
+     * Returns the given string if it is non-null; the empty string otherwise.
+     */
+    public static String nullToEmpty(final String string) {
+        return (string == null) ? "" : string;
+    }
+
+    /**
+     * Returns the given string if it is nonempty; {@code null} otherwise.
+     */
+    public static String emptyToNull(final String string) {
+        return isNullOrEmpty(string) ? null : string;
+    }
+
+    /**
+     * Returns {@code true} if the given string is null or is the empty string.
+     */
+    public static boolean isNullOrEmpty(final String str) {
+        return str == null || str.length() == 0;
+    }
+
+    /**
+     * Checks if a string is whitespace, empty ("") or null.
+     *
+     * Strings.isBlank(null)      = true
+     * Strings.isBlank("")        = true
+     * Strings.isBlank(" ")       = true
+     * Strings.isBlank("bob")     = false
+     * Strings.isBlank("  bob  ") = false
+     */
+    public static boolean isBlank(final String str) {
+        final int strLen;
+        if (str != null && (strLen = str.length()) != 0) {
+            for (int i = 0; i < strLen; i++) {
+                if (!Character.isWhitespace(str.charAt(i))) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Checks if a string is not empty (""), not null and not whitespace only.
+     *
+     * Strings.isNotBlank(null)      = false
+     * Strings.isNotBlank("")        = false
+     * Strings.isNotBlank(" ")       = false
+     * Strings.isNotBlank("bob")     = true
+     * Strings.isNotBlank("  bob  ") = true
+     */
+    public static boolean isNotBlank(final String str) {
+        return !isBlank(str);
+    }
+
+    /**
+     * Splits the provided text into an array, separator specified.
+     *
+     * A null input String returns null.
+     *
+     * Strings.split(null, *)         = null
+     * Strings.split("", *)           = []
+     * Strings.split("a.b.c", '.')    = ["a", "b", "c"]
+     * Strings.split("a..b.c", '.')   = ["a", "b", "c"]
+     * Strings.split("a:b:c", '.')    = ["a:b:c"]
+     * Strings.split("a b c", ' ')    = ["a", "b", "c"]
+     */
+    public static String[] split(final String str, final char separator) {
+        return split(str, separator, false);
+    }
+
+    /**
+     * Splits the provided text into an array, separator specified,
+     * if {@code} true, preserving all tokens, including empty tokens created
+     * by adjacent separators.
+     *
+     * A null input String returns null.
+     *
+     * Strings.split(null, *, true)         = null
+     * Strings.split("", *, true)           = []
+     * Strings.split("a.b.c", '.', true)    = ["a", "b", "c"]
+     * Strings.split("a..b.c", '.', true)   = ["a", "", "b", "c"]
+     * Strings.split("a:b:c", '.', true)    = ["a:b:c"]
+     * Strings.split("a b c", ' ', true)    = ["a", "b", "c"]
+     * Strings.split("a b c ", ' ', true)   = ["a", "b", "c", ""]
+     * Strings.split("a b c  ", ' ', true)  = ["a", "b", "c", "", ""]
+     * Strings.split(" a b c", ' ', true)   = ["", a", "b", "c"]
+     * Strings.split("  a b c", ' ', true)  = ["", "", a", "b", "c"]
+     * Strings.split(" a b c ", ' ', true)  = ["", a", "b", "c", ""]
+     */
+    public static String[] split(final String str, final char separator, final boolean preserveAllTokens) {
+        if (str == null) {
+            return null;
+        }
+        final int len = str.length();
+        if (len == 0) {
+            return EMPTY_STRING_ARRAY;
+        }
+        final List list = Lists.newArrayList();
+        int i = 0, start = 0;
+        boolean match = false;
+        while (i < len) {
+            if (str.charAt(i) == separator) {
+                if (match || preserveAllTokens) {
+                    list.add(str.substring(start, i));
+                    match = false;
+                }
+                start = ++i;
+                continue;
+            }
+            match = true;
+            i++;
+        }
+        if (match || preserveAllTokens) {
+            list.add(str.substring(start, i));
+        }
+        return list.toArray(new String[0]);
+    }
+
+    private static final String[] EMPTY_STRING_ARRAY = new String[0];
+
+    private Strings() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/UniqueIdUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/UniqueIdUtil.java
new file mode 100644
index 0000000..d1a5ef8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/UniqueIdUtil.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class UniqueIdUtil {
+
+    private static final Logger     logger         = LoggerFactory.getLogger(UniqueIdUtil.class);
+
+    // maximal value for 64bit systems is 2^22, see man 5 proc.
+    private static final int        MAX_PROCESS_ID = 4194304;
+    private static final char       PID_FLAG       = 'd';
+    private static final String     IP_16;
+    private static final String     PID;
+    private static final long       ID_BASE        = 1000;
+    private static final long       ID_MASK        = (1 << 13) - 1;                              // 8192 - 1
+    private static final AtomicLong sequence       = new AtomicLong();
+
+    static {
+        String ip16;
+        try {
+            final String ip = NetUtil.getLocalAddress();
+            ip16 = getIp16(ip);
+        } catch (final Throwable t) {
+            ip16 = "ffffffff";
+        }
+        IP_16 = ip16;
+
+        String pid;
+        try {
+            pid = getHexProcessId(getProcessId());
+        } catch (final Throwable t) {
+            pid = "0000";
+        }
+        PID = pid;
+    }
+
+    public static String generateId() {
+        return getId(IP_16, Clock.defaultClock().getTime(), getNextId());
+    }
+
+    private static String getHexProcessId(int pid) {
+        // unsigned short 0 to 65535
+        if (pid < 0) {
+            pid = 0;
+        }
+        if (pid > 65535) {
+            String strPid = Integer.toString(pid);
+            strPid = strPid.substring(strPid.length() - 4);
+            pid = Integer.parseInt(strPid);
+        }
+        final StringBuilder buf = new StringBuilder(Integer.toHexString(pid));
+        while (buf.length() < 4) {
+            buf.insert(0, "0");
+        }
+        return buf.toString();
+    }
+
+    /**
+     * Gets current pid, max pid 32 bit systems 32768, for 64 bit 4194304
+     * http://unix.stackexchange.com/questions/16883/what-is-the-maximum-value-of-the-pid-of-a-process
+     * http://stackoverflow.com/questions/35842/how-can-a-java-program-get-its-own-process-id
+     */
+    private static int getProcessId() {
+        String value = "";
+        try {
+            final RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
+            value = runtime.getName();
+        } catch (final Throwable t) {
+            if (logger.isDebugEnabled()) {
+                logger.debug("Could not invoke ManagementFactory.getRuntimeMXBean().getName(), {}.",
+                    StackTraceUtil.stackTrace(t));
+            }
+        }
+
+        // something like '@', at least in SUN / Oracle JVMs
+        final int atIndex = value.indexOf('@');
+        if (atIndex >= 0) {
+            value = value.substring(0, atIndex);
+        }
+
+        int pid = -1;
+        try {
+            pid = Integer.parseInt(value);
+        } catch (final NumberFormatException ignored) {
+            // value did not contain an integer
+        }
+
+        if (pid < 0 || pid > MAX_PROCESS_ID) {
+            pid = ThreadLocalRandom.current().nextInt(MAX_PROCESS_ID + 1);
+
+            logger.warn("Failed to find the current process ID from '{}'; using a random value: {}.", value, pid);
+        }
+
+        return pid;
+    }
+
+    private static String getIp16(final String ip) {
+        final String[] segments = ip.split("\\.");
+        final StringBuilder buf = StringBuilderHelper.get();
+        for (final String s : segments) {
+            final String hex = Integer.toHexString(Integer.parseInt(s));
+            if (hex.length() == 1) {
+                buf.append('0');
+            }
+            buf.append(hex);
+        }
+        return buf.toString();
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private static String getId(final String ip16, final long timestamp, final long nextId) {
+        return StringBuilderHelper.get() //
+            .append(ip16) //
+            .append(timestamp) //
+            .append(nextId) //
+            .append(PID_FLAG) //
+            .append(PID) //
+            .toString();
+    }
+
+    private static long getNextId() {
+        // (1000 + 1) ~ (1000 + 8191)
+        return (sequence.incrementAndGet() & ID_MASK) + ID_BASE;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/VarInts.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/VarInts.java
new file mode 100644
index 0000000..619a0b9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/VarInts.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("all")
+public final class VarInts {
+
+    public static byte[] writeVarInt32(int value) {
+        int position = 0;
+        int size = computeRawVarInt32Size(value);
+        byte[] bytes = new byte[size];
+        while (true) {
+            if ((value & ~0x7F) == 0) {
+                bytes[position] = (byte) value;
+                return bytes;
+            } else {
+                bytes[position++] = (byte) ((value & 0x7F) | 0x80);
+                value >>>= 7;
+            }
+        }
+    }
+
+    public static int readVarInt32(byte[] bytes) {
+        int position = 0;
+        byte tmp = bytes[position++];
+        if (tmp >= 0) {
+            return tmp;
+        }
+        int result = tmp & 0x7f;
+        if ((tmp = bytes[position++]) >= 0) {
+            result |= tmp << 7;
+        } else {
+            result |= (tmp & 0x7f) << 7;
+            if ((tmp = bytes[position++]) >= 0) {
+                result |= tmp << 14;
+            } else {
+                result |= (tmp & 0x7f) << 14;
+                if ((tmp = bytes[position++]) >= 0) {
+                    result |= tmp << 21;
+                } else {
+                    result |= (tmp & 0x7f) << 21;
+                    result |= (tmp = bytes[position++]) << 28;
+                    if (tmp < 0) {
+                        // Discard upper 32 bits.
+                        for (int i = 0; i < 5; i++) {
+                            if (bytes[position++] >= 0) {
+                                return result;
+                            }
+                        }
+                        throw new RuntimeException("encountered a malformed varInt");
+                    }
+                }
+            }
+        }
+        return result;
+    }
+
+    public static byte[] writeVarInt64(long value) {
+        int position = 0;
+        int size = computeRawVarInt64Size(value);
+        byte[] bytes = new byte[size];
+        while (true) {
+            if ((value & ~0x7FL) == 0) {
+                bytes[position] = (byte) value;
+                return bytes;
+            } else {
+                bytes[position++] = (byte) (((int) value & 0x7F) | 0x80);
+                value >>>= 7;
+            }
+        }
+    }
+
+    public static long readVarInt64(byte[] bytes) {
+        int shift = 0;
+        long result = 0;
+        int position = 0;
+        while (shift < 64) {
+            final byte b = bytes[position++];
+            result |= (long) (b & 0x7F) << shift;
+            if ((b & 0x80) == 0) {
+                return result;
+            }
+            shift += 7;
+        }
+        throw new RuntimeException("encountered a malformed varInt");
+    }
+
+    /**
+     * Compute the number of bytes that would be needed to encode a varInt.
+     * {@code value} is treated as unsigned, so it won't be sign-extended
+     * if negative.
+     */
+    public static int computeRawVarInt32Size(final int value) {
+        if ((value & (~0 << 7)) == 0) {
+            return 1;
+        }
+        if ((value & (~0 << 14)) == 0) {
+            return 2;
+        }
+        if ((value & (~0 << 21)) == 0) {
+            return 3;
+        }
+        if ((value & (~0 << 28)) == 0) {
+            return 4;
+        }
+        return 5;
+    }
+
+    /**
+     * Compute the number of bytes that would be needed to encode a varInt.
+     */
+    public static int computeRawVarInt64Size(long value) {
+        // Handle two popular special cases up front ...
+        if ((value & (~0L << 7)) == 0L) {
+            return 1;
+        }
+        if (value < 0L) {
+            return 10;
+        }
+        // ... leaving us with 8 remaining, which we can divide and conquer
+        int n = 2;
+        if ((value & (~0L << 35)) != 0L) {
+            n += 4;
+            value >>>= 28;
+        }
+        if ((value & (~0L << 21)) != 0L) {
+            n += 2;
+            value >>>= 14;
+        }
+        if ((value & (~0L << 14)) != 0L) {
+            n += 1;
+        }
+        return n;
+    }
+
+    private VarInts() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ZipUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ZipUtil.java
new file mode 100644
index 0000000..d630f53
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/ZipUtil.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.zip.CheckedInputStream;
+import java.util.zip.CheckedOutputStream;
+import java.util.zip.Checksum;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
+
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ZipUtil {
+
+    public static void compress(final String rootDir, final String sourceDir, final String outputFile,
+                                final Checksum checksum) throws IOException {
+        try (final FileOutputStream fos = new FileOutputStream(outputFile);
+                final CheckedOutputStream cos = new CheckedOutputStream(fos, checksum);
+                final ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(cos))) {
+            ZipUtil.compressDirectoryToZipFile(rootDir, sourceDir, zos);
+            zos.flush();
+            fos.getFD().sync();
+        }
+        Utils.fsync(new File(outputFile));
+    }
+
+    private static void compressDirectoryToZipFile(final String rootDir, final String sourceDir,
+                                                   final ZipOutputStream zos) throws IOException {
+        final String dir = Paths.get(rootDir, sourceDir).toString();
+        final File[] files = Requires.requireNonNull(new File(dir).listFiles(), "files");
+        for (final File file : files) {
+            final String child = Paths.get(sourceDir, file.getName()).toString();
+            if (file.isDirectory()) {
+                compressDirectoryToZipFile(rootDir, child, zos);
+            } else {
+                zos.putNextEntry(new ZipEntry(child));
+                try (final FileInputStream fis = new FileInputStream(file);
+                        final BufferedInputStream bis = new BufferedInputStream(fis)) {
+                    IOUtils.copy(bis, zos);
+                }
+            }
+        }
+    }
+
+    public static void decompress(final String sourceFile, final String outputDir, final Checksum checksum)
+                                                                                                           throws IOException {
+        try (final FileInputStream fis = new FileInputStream(sourceFile);
+                final CheckedInputStream cis = new CheckedInputStream(fis, checksum);
+                final ZipInputStream zis = new ZipInputStream(new BufferedInputStream(cis))) {
+            ZipEntry entry;
+            while ((entry = zis.getNextEntry()) != null) {
+                final String fileName = entry.getName();
+                final File entryFile = new File(Paths.get(outputDir, fileName).toString());
+                FileUtils.forceMkdir(entryFile.getParentFile());
+                try (final FileOutputStream fos = new FileOutputStream(entryFile);
+                        final BufferedOutputStream bos = new BufferedOutputStream(fos)) {
+                    IOUtils.copy(zis, bos);
+                    bos.flush();
+                    fos.getFD().sync();
+                }
+            }
+            // Continue to read all remaining bytes(extra metadata of ZipEntry) directly from the checked stream,
+            // Otherwise, the checksum value maybe unexpected.
+            //
+            // See https://coderanch.com/t/279175/java/ZipInputStream
+            IOUtils.copy(cis, NullOutputStream.NULL_OUTPUT_STREAM);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AbstractRejectedExecutionHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AbstractRejectedExecutionHandler.java
new file mode 100644
index 0000000..a646610
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AbstractRejectedExecutionHandler.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.JvmTools;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class AbstractRejectedExecutionHandler implements RejectedExecutionHandler {
+
+    protected static final Logger LOG = LoggerFactory.getLogger(AbstractRejectedExecutionHandler.class);
+
+    protected final String        threadPoolName;
+    private final AtomicBoolean   dumpNeeded;
+    private final String          dumpPrefixName;
+
+    public AbstractRejectedExecutionHandler(String threadPoolName, boolean dumpNeeded, String dumpPrefixName) {
+        this.threadPoolName = threadPoolName;
+        this.dumpNeeded = new AtomicBoolean(dumpNeeded);
+        this.dumpPrefixName = dumpPrefixName;
+    }
+
+    public void dumpJvmInfoIfNeeded() {
+        if (this.dumpNeeded.getAndSet(false)) {
+            final String now = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date());
+            final String name = this.threadPoolName + "_" + now;
+            try (final FileOutputStream fileOutput = new FileOutputStream(new File(this.dumpPrefixName + "_dump_"
+                                                                                   + name + ".log"))) {
+
+                final List stacks = JvmTools.jStack();
+                for (final String s : stacks) {
+                    fileOutput.write(Utils.getBytes(s));
+                }
+
+                final List memoryUsages = JvmTools.memoryUsage();
+                for (final String m : memoryUsages) {
+                    fileOutput.write(Utils.getBytes(m));
+                }
+
+                if (JvmTools.memoryUsed() > 0.9) {
+                    JvmTools.jMap(this.dumpPrefixName + "_dump_" + name + ".bin", false);
+                }
+            } catch (final Throwable t) {
+                LOG.error("Dump jvm info error: {}.", StackTraceUtil.stackTrace(t));
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AffinityNamedThreadFactory.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AffinityNamedThreadFactory.java
new file mode 100644
index 0000000..c43bc2f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/AffinityNamedThreadFactory.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import net.openhft.affinity.AffinityLock;
+import net.openhft.affinity.AffinityStrategies;
+import net.openhft.affinity.AffinityStrategy;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * This is a ThreadFactory which assigns threads based the strategies provided.
+ *
+ * If no strategies are provided AffinityStrategies.ANY is used.
+ *
+ * @author jiachun.fjc
+ */
+public class AffinityNamedThreadFactory implements ThreadFactory {
+
+    private static final Logger      LOG              = LoggerFactory.getLogger(AffinityNamedThreadFactory.class);
+
+    private final AtomicInteger      id               = new AtomicInteger();
+    private final String             name;
+    private final boolean            daemon;
+    private final int                priority;
+    private final ThreadGroup        group;
+    private final AffinityStrategy[] strategies;
+    private AffinityLock             lastAffinityLock = null;
+
+    public AffinityNamedThreadFactory(String name, AffinityStrategy... strategies) {
+        this(name, false, Thread.NORM_PRIORITY, strategies);
+    }
+
+    public AffinityNamedThreadFactory(String name, boolean daemon, AffinityStrategy... strategies) {
+        this(name, daemon, Thread.NORM_PRIORITY, strategies);
+    }
+
+    public AffinityNamedThreadFactory(String name, int priority, AffinityStrategy... strategies) {
+        this(name, false, priority, strategies);
+    }
+
+    public AffinityNamedThreadFactory(String name, boolean daemon, int priority, AffinityStrategy... strategies) {
+        this.name = "affinity." + name + " #";
+        this.daemon = daemon;
+        this.priority = priority;
+        final SecurityManager s = System.getSecurityManager();
+        this.group = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup();
+        this.strategies = strategies.length == 0 ? new AffinityStrategy[] { AffinityStrategies.ANY } : strategies;
+    }
+
+    @SuppressWarnings("all")
+    @Override
+    public Thread newThread(final Runnable r) {
+        Requires.requireNonNull(r, "runnable");
+
+        final String name2 = this.name + this.id.getAndIncrement();
+        final Runnable r2 = wrapRunnable(r);
+        final Runnable r3 = new Runnable() {
+
+            @Override
+            public void run() {
+                AffinityLock al = null;
+                try {
+                    al = acquireLockBasedOnLast();
+                } catch (final Throwable ignored) {
+                    // Defensive: ignored error on acquiring lock
+                }
+                try {
+                    r2.run();
+                } finally {
+                    if (al != null) {
+                        try {
+                            al.release();
+                        } catch (final Throwable ignored) {
+                            // Defensive: ignored error on releasing lock
+                        }
+                    }
+                }
+            }
+        };
+
+        final Thread t = wrapThread(this.group, r3, name2);
+
+        try {
+            if (t.isDaemon() != this.daemon) {
+                t.setDaemon(this.daemon);
+            }
+
+            if (t.getPriority() != this.priority) {
+                t.setPriority(this.priority);
+            }
+        } catch (final Exception ignored) {
+            // Doesn't matter even if failed to set.
+        }
+
+        LOG.info("Creates new {}.", t);
+
+        return t;
+    }
+
+    public ThreadGroup getThreadGroup() {
+        return group;
+    }
+
+    protected Runnable wrapRunnable(final Runnable r) {
+        return r; // InternalThreadLocalRunnable.wrap(r)
+    }
+
+    protected Thread wrapThread(final ThreadGroup group, final Runnable r, final String name) {
+        return new Thread(group, r, name);
+    }
+
+    private synchronized AffinityLock acquireLockBasedOnLast() {
+        final AffinityLock al = this.lastAffinityLock == null ? AffinityLock.acquireLock() : lastAffinityLock
+            .acquireLock(this.strategies);
+        if (al.cpuId() >= 0) {
+            if (!al.isBound()) {
+                al.bind();
+            }
+            this.lastAffinityLock = al;
+        }
+        return al;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/BlockingProducersPolicyWithReport.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/BlockingProducersPolicyWithReport.java
new file mode 100644
index 0000000..f642ea9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/BlockingProducersPolicyWithReport.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * A handler for rejected tasks that use blocking producer policy,
+ * not discard task, not throw exception, when the queue is full,
+ * call 'BlockingQueue.put' to let the producer blocking.
+ *
+ * @author jiachun.fjc
+ */
+public class BlockingProducersPolicyWithReport extends AbstractRejectedExecutionHandler {
+
+    public BlockingProducersPolicyWithReport(String threadPoolName) {
+        super(threadPoolName, false, "");
+    }
+
+    public BlockingProducersPolicyWithReport(String threadPoolName, String dumpPrefixName) {
+        super(threadPoolName, true, dumpPrefixName);
+    }
+
+    @Override
+    public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+        LOG.error("Thread pool [{}] is exhausted! {}.", threadPoolName, e.toString());
+
+        dumpJvmInfoIfNeeded();
+
+        if (!e.isShutdown()) {
+            try {
+                e.getQueue().put(r);
+            } catch (InterruptedException ignored) {
+                // Should not be interrupted
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/CallerRunsPolicyWithReport.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/CallerRunsPolicyWithReport.java
new file mode 100644
index 0000000..d8996cc
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/CallerRunsPolicyWithReport.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * A handler for rejected tasks that use the caller-runs policy,
+ * not discard task, not throw exception, let caller thread to
+ * execute the task.
+ *
+ * @author jiachun.fjc
+ */
+public class CallerRunsPolicyWithReport extends AbstractRejectedExecutionHandler {
+
+    public CallerRunsPolicyWithReport(String threadPoolName) {
+        super(threadPoolName, false, "");
+    }
+
+    public CallerRunsPolicyWithReport(String threadPoolName, String dumpPrefixName) {
+        super(threadPoolName, true, dumpPrefixName);
+    }
+
+    @Override
+    public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+        LOG.error("Thread pool [{}] is exhausted! {}.", threadPoolName, e.toString());
+
+        dumpJvmInfoIfNeeded();
+
+        if (!e.isShutdown()) {
+            r.run();
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardOldPolicyWithReport.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardOldPolicyWithReport.java
new file mode 100644
index 0000000..cb5f586
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardOldPolicyWithReport.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * A handler for rejected tasks that discards the oldest unhandled
+ * request and then retries {@code execute}, unless the executor
+ * is shut down, in which case the task is discarded.
+ *
+ * @author jiachun.fjc
+ */
+public class DiscardOldPolicyWithReport extends AbstractRejectedExecutionHandler {
+
+    public DiscardOldPolicyWithReport(String threadPoolName) {
+        super(threadPoolName, false, "");
+    }
+
+    public DiscardOldPolicyWithReport(String threadPoolName, String dumpPrefixName) {
+        super(threadPoolName, true, dumpPrefixName);
+    }
+
+    @Override
+    public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+        LOG.error("Thread pool [{}] is exhausted! {}.", threadPoolName, e.toString());
+
+        dumpJvmInfoIfNeeded();
+
+        if (!e.isShutdown()) {
+            final BlockingQueue queue = e.getQueue();
+            int discardSize = queue.size() >> 1;
+            discardSize = discardSize <= 0 ? 1 : discardSize;
+            for (int i = 0; i < discardSize; i++) {
+                queue.poll();
+            }
+            e.execute(r);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardPolicyWithReport.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardPolicyWithReport.java
new file mode 100644
index 0000000..5483149
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DiscardPolicyWithReport.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * A handler for rejected tasks that silently discards the
+ * rejected task.
+ *
+ * @author jiachun.fjc
+ */
+public class DiscardPolicyWithReport extends AbstractRejectedExecutionHandler {
+
+    public DiscardPolicyWithReport(String threadPoolName) {
+        super(threadPoolName, false, "");
+    }
+
+    @Override
+    public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+        LOG.error("Thread pool [{}] is exhausted! {}.", threadPoolName, e.toString());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DistributedLock.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DistributedLock.java
new file mode 100644
index 0000000..ddc3e99
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/DistributedLock.java
@@ -0,0 +1,406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.io.Serializable;
+import java.util.Objects;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.rhea.util.UniqueIdUtil;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * A distributed lock that provides exclusive access to a
+ * shared resource.
+ *
+ * 
+ *      DistributedLock lock = ...;
+ *      if (lock.tryLock()) {
+ *          try {
+ *              // manipulate protected state
+ *          } finally {
+ *              lock.unlock();
+ *          }
+ *      } else {
+ *          // perform alternative actions
+ *      }
+ * 

+ *
+ * @author jiachun.fjc
+ */
+public abstract class DistributedLock {
+
+    private final T                        internalKey;
+    private final Acquirer                 acquirer;
+    private final ScheduledExecutorService watchdog;
+
+    private volatile Owner                 owner;
+
+    protected DistributedLock(T target, long lease, TimeUnit unit, ScheduledExecutorService watchdog) {
+        Requires.requireTrue(lease >= 0, "lease must >= 0");
+        this.internalKey = withInternalKey(target);
+        this.acquirer = new Acquirer(UniqueIdUtil.generateId(), unit.toMillis(lease));
+        this.watchdog = watchdog;
+    }
+
+    /**
+     * @see #tryLock(byte[])
+     */
+    public boolean tryLock() {
+        return tryLock(null);
+    }
+
+    /**
+     * Acquires the lock only if it is free at the time of invocation.
+     *
+     * Acquires the lock if it is available and returns immediately
+     * with the value {@code true}.
+     * If the lock is not available then this method will return
+     * immediately with the value {@code false}.
+     *
+     * @param ctx the context of current lock request
+     * @return {@code true} if the lock was acquired and {@code false}
+     * otherwise
+     */
+    public boolean tryLock(final byte[] ctx) {
+        return internalTryLock(ctx).isSuccess();
+    }
+
+    /**
+     * @see #tryLock(byte[], long, TimeUnit)
+     */
+    public boolean tryLock(final long timeout, final TimeUnit unit) {
+        return tryLock(null, timeout, unit);
+    }
+
+    /**
+     * Acquires the lock if it is free within the given waiting time.
+     *
+     * If the lock is available this method returns immediately
+     * with the value {@code true}.
+     * If the lock is not available then
+     * the current thread becomes disabled for thread scheduling
+     * purposes and lies dormant until one of two things happens:
+     * 
    
+     * 
  • The lock is acquired by the current thread; or
+     * 
  • The specified waiting time elapses
+     *
+     * @param ctx     the context of current lock request
+     * @param timeout the maximum time to wait for the lock
+     * @param unit    the time unit of the {@code time} argument
+     * @return {@code true} if the lock was acquired and {@code false}
+     * if the waiting time elapsed before the lock was acquired
+     */
+    public boolean tryLock(final byte[] ctx, final long timeout, final TimeUnit unit) {
+        final long timeoutNs = unit.toNanos(timeout);
+        final long startNs = System.nanoTime();
+        int attempts = 1;
+        try {
+            for (;;) {
+                final Owner owner = internalTryLock(ctx);
+                if (owner.isSuccess()) {
+                    return true;
+                }
+                if (System.nanoTime() - startNs >= timeoutNs) {
+                    break;
+                }
+                if (attempts < 8) {
+                    attempts++;
+                }
+                final long remaining = Math.max(0, owner.getRemainingMillis());
+                // TODO optimize with notify?
+                // avoid liveLock
+                Thread.sleep(Math.min(remaining, 2 << attempts));
+            }
+        } catch (final Throwable t) {
+            ThrowUtil.throwException(t);
+        }
+        return false;
+    }
+
+    /**
+     * Attempts to release this lock.
+     *
+     * If the current caller is the holder of this lock then the hold
+     * count is decremented.  If the hold count is now zero then the
+     * lock is released.  If the current caller is not the holder of
+     * this lock then InvalidLockAcquirerException is thrown.
+     */
+    public abstract void unlock();
+
+    /**
+     * Making the lock safe with fencing token.
+     *
+     * Is simply a number that increases (e.g. incremented by the lock
+     * service) every time a client acquires the lock.
+     */
+    public long getFencingToken() {
+        return this.acquirer.getFencingToken();
+    }
+
+    /**
+     * Returns the 'lock-context' of current lock owner.
+     */
+    public byte[] getOwnerContext() {
+        return getOwner().getContext();
+    }
+
+    public ScheduledExecutorService getWatchdog() {
+        return watchdog;
+    }
+
+    public Acquirer getAcquirer() {
+        return acquirer;
+    }
+
+    public Owner getOwner() {
+        final Owner copy = this.owner;
+        if (copy == null) {
+            throw new IllegalStateException("must try to lock at first");
+        }
+        return copy;
+    }
+
+    public static OwnerBuilder newOwnerBuilder() {
+        return new OwnerBuilder();
+    }
+
+    protected abstract Owner internalTryLock(final byte[] ctx);
+
+    protected T withInternalKey(final T target) {
+        // override this method to plastic with target, default do nothing
+        return target;
+    }
+
+    protected T getInternalKey() {
+        return internalKey;
+    }
+
+    protected void updateOwner(final Owner owner) {
+        this.owner = owner;
+    }
+
+    protected void updateOwnerAndAcquirer(final Owner owner) {
+        this.owner = owner;
+        if (this.owner != null) {
+            this.owner.updateAcquirerInfo(this.acquirer);
+        }
+    }
+
+    public static class Acquirer implements Serializable {
+
+        private static final long serialVersionUID = -9174459539789423607L;
+
+        private final String      id;
+        private final long        leaseMillis;
+
+        // the time on trying to lock, it must be set by lock server
+        private volatile long     lockingTimestamp;
+        // making the lock safe with fencing token.
+        //
+        // is simply a number that increases (e.g. incremented by the lock service)
+        // every time a client acquires the lock.
+        private volatile long     fencingToken;
+        // the context of current lock request
+        private volatile byte[]   context;
+
+        public Acquirer(String id, long leaseMillis) {
+            this.id = id;
+            this.leaseMillis = leaseMillis;
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public long getLeaseMillis() {
+            return leaseMillis;
+        }
+
+        public long getLockingTimestamp() {
+            return lockingTimestamp;
+        }
+
+        public void setLockingTimestamp(long lockingTimestamp) {
+            this.lockingTimestamp = lockingTimestamp;
+        }
+
+        public long getFencingToken() {
+            return fencingToken;
+        }
+
+        public void setFencingToken(long fencingToken) {
+            this.fencingToken = fencingToken;
+        }
+
+        public byte[] getContext() {
+            return context;
+        }
+
+        public void setContext(byte[] context) {
+            this.context = context;
+        }
+
+        @Override
+        public String toString() {
+            return "Acquirer{" + "id='" + id + '\'' + ", leaseMillis=" + leaseMillis + ", lockingTimestamp="
+                   + lockingTimestamp + ", fencingToken=" + fencingToken + ", context=" + BytesUtil.toHex(context)
+                   + '}';
+        }
+    }
+
+    public static class Owner implements Serializable {
+
+        private static final long serialVersionUID = 3939239434225894164L;
+
+        // locker id
+        private final String      id;
+        // absolute time for this lock to expire
+        private final long        deadlineMillis;
+        // remainingMillis < 0 means lock successful
+        private final long        remainingMillis;
+        // making the lock safe with fencing token
+        //
+        // is simply a number that increases (e.g. incremented by the lock service)
+        // every time a client acquires the lock.
+        private final long        fencingToken;
+        // for reentrant lock
+        private final long        acquires;
+        // the context of current lock owner
+        private final byte[]      context;
+        // if operation success
+        private final boolean     success;
+
+        public Owner(String id, long deadlineMillis, long remainingMillis, long fencingToken, long acquires,
+                     byte[] context, boolean success) {
+            this.id = id;
+            this.deadlineMillis = deadlineMillis;
+            this.remainingMillis = remainingMillis;
+            this.fencingToken = fencingToken;
+            this.acquires = acquires;
+            this.context = context;
+            this.success = success;
+        }
+
+        public boolean isSameAcquirer(final Acquirer acquirer) {
+            return acquirer != null && this.fencingToken == acquirer.fencingToken
+                   && Objects.equals(this.id, acquirer.id);
+        }
+
+        public void updateAcquirerInfo(final Acquirer acquirer) {
+            if (acquirer == null) {
+                return;
+            }
+            acquirer.setFencingToken(this.fencingToken);
+        }
+
+        public String getId() {
+            return id;
+        }
+
+        public long getDeadlineMillis() {
+            return deadlineMillis;
+        }
+
+        public long getRemainingMillis() {
+            return remainingMillis;
+        }
+
+        public long getFencingToken() {
+            return fencingToken;
+        }
+
+        public long getAcquires() {
+            return acquires;
+        }
+
+        public byte[] getContext() {
+            return context;
+        }
+
+        public boolean isSuccess() {
+            return success;
+        }
+
+        @Override
+        public String toString() {
+            return "Owner{" + "id='" + id + '\'' + ", deadlineMillis=" + deadlineMillis + ", remainingMillis="
+                   + remainingMillis + ", fencingToken=" + fencingToken + ", acquires=" + acquires + ", context="
+                   + BytesUtil.toHex(context) + ", success=" + success + '}';
+        }
+    }
+
+    public static class OwnerBuilder {
+
+        public static long KEEP_LEASE_FAIL     = Long.MAX_VALUE;
+        public static long FIRST_TIME_SUCCESS  = -1;
+        public static long NEW_ACQUIRE_SUCCESS = -2;
+        public static long KEEP_LEASE_SUCCESS  = -3;
+        public static long REENTRANT_SUCCESS   = -4;
+
+        private String     id;
+        private long       deadlineMillis;
+        private long       remainingMillis;
+        private long       fencingToken;
+        private long       acquires;
+        private byte[]     context;
+        private boolean    success;
+
+        public Owner build() {
+            return new Owner(this.id, this.deadlineMillis, this.remainingMillis, this.fencingToken, this.acquires,
+                this.context, this.success);
+        }
+
+        public OwnerBuilder id(final String id) {
+            this.id = id;
+            return this;
+        }
+
+        public OwnerBuilder deadlineMillis(final long deadlineMillis) {
+            this.deadlineMillis = deadlineMillis;
+            return this;
+        }
+
+        public OwnerBuilder remainingMillis(final long remainingMillis) {
+            this.remainingMillis = remainingMillis;
+            return this;
+        }
+
+        public OwnerBuilder fencingToken(final long fencingToken) {
+            this.fencingToken = fencingToken;
+            return this;
+        }
+
+        public OwnerBuilder acquires(final long acquires) {
+            this.acquires = acquires;
+            return this;
+        }
+
+        public OwnerBuilder context(final byte[] context) {
+            this.context = context;
+            return this;
+        }
+
+        public OwnerBuilder success(final boolean success) {
+            this.success = success;
+            return this;
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/NamedThreadFactory.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/NamedThreadFactory.java
new file mode 100644
index 0000000..9c4b84e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/NamedThreadFactory.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * Named thread factory.
+ *
+ * @author jiachun.fjc
+ */
+public class NamedThreadFactory implements ThreadFactory {
+
+    private static final Logger LOG = LoggerFactory.getLogger(NamedThreadFactory.class);
+
+    private final AtomicInteger id  = new AtomicInteger();
+    private final String        name;
+    private final boolean       daemon;
+    private final int           priority;
+    private final ThreadGroup   group;
+
+    public NamedThreadFactory(String name) {
+        this(name, false, Thread.NORM_PRIORITY);
+    }
+
+    public NamedThreadFactory(String name, boolean daemon) {
+        this(name, daemon, Thread.NORM_PRIORITY);
+    }
+
+    public NamedThreadFactory(String name, int priority) {
+        this(name, false, priority);
+    }
+
+    public NamedThreadFactory(String name, boolean daemon, int priority) {
+        this.name = name + " #";
+        this.daemon = daemon;
+        this.priority = priority;
+        final SecurityManager s = System.getSecurityManager();
+        this.group = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup();
+    }
+
+    @Override
+    public Thread newThread(final Runnable r) {
+        Requires.requireNonNull(r, "runnable");
+
+        final String name2 = this.name + this.id.getAndIncrement();
+        final Runnable r2 = wrapRunnable(r);
+        final Thread t = wrapThread(this.group, r2, name2);
+
+        try {
+            if (t.isDaemon() != this.daemon) {
+                t.setDaemon(this.daemon);
+            }
+
+            if (t.getPriority() != this.priority) {
+                t.setPriority(this.priority);
+            }
+        } catch (final Exception ignored) {
+            // Doesn't matter even if failed to set.
+        }
+
+        LOG.info("Creates new {}.", t);
+
+        return t;
+    }
+
+    public ThreadGroup getThreadGroup() {
+        return group;
+    }
+
+    protected Runnable wrapRunnable(final Runnable r) {
+        return r; // InternalThreadLocalRunnable.wrap(r)
+    }
+
+    protected Thread wrapThread(final ThreadGroup group, final Runnable r, final String name) {
+        return new Thread(group, r, name);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedRunnable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedRunnable.java
new file mode 100644
index 0000000..0f80f34
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedRunnable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface RejectedRunnable extends Runnable {
+
+    void rejected();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedTaskPolicyWithReport.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedTaskPolicyWithReport.java
new file mode 100644
index 0000000..8662d18
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/RejectedTaskPolicyWithReport.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+
+/**
+ * If the current task implements {@link RejectedRunnable},
+ * then the users will be allowed to implement user-defined
+ * rejecting, otherwise some of the existing tasks in the
+ * queue will be discarded in FIFO mode.
+ *
+ * @author jiachun.fjc
+ */
+public class RejectedTaskPolicyWithReport extends AbstractRejectedExecutionHandler {
+
+    public RejectedTaskPolicyWithReport(String threadPoolName) {
+        super(threadPoolName, false, "");
+    }
+
+    public RejectedTaskPolicyWithReport(String threadPoolName, String dumpPrefixName) {
+        super(threadPoolName, true, dumpPrefixName);
+    }
+
+    @Override
+    public void rejectedExecution(final Runnable r, final ThreadPoolExecutor e) {
+        LOG.error("Thread pool [{}] is exhausted! {}.", threadPoolName, e.toString());
+
+        dumpJvmInfoIfNeeded();
+
+        if (r instanceof RejectedRunnable) {
+            ((RejectedRunnable) r).rejected(); // user-defined rejecting
+        } else {
+            if (!e.isShutdown()) {
+                final BlockingQueue queue = e.getQueue();
+                final int discardSize = queue.size() >> 1;
+                for (int i = 0; i < discardSize; i++) {
+                    queue.poll();
+                }
+
+                try {
+                    queue.put(r);
+                } catch (final InterruptedException ignored) {
+                    // should not be interrupted
+                }
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/AbstractEntry.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/AbstractEntry.java
new file mode 100644
index 0000000..a067a5c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/AbstractEntry.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.collection;
+
+import java.util.Map;
+
+/**
+ * A simple implementation of {@link Map.Entry}.
+ * Does not implement {@link Map.Entry#setValue(Object)}, that is done by users of the class.
+ *
+ * @param  the type of keys maintained by this map
+ * @param  the type of mapped values
+ * @author Cliff Click
+ * @since 1.5
+ *
+ * Forked from JCTools.
+ */
+abstract class AbstractEntry implements Map.Entry {
+    /**
+     * Strongly typed key
+     */
+    protected final TypeK _key;
+    /**
+     * Strongly typed value
+     */
+    protected TypeV       _val;
+
+    public AbstractEntry(final TypeK key, final TypeV val) {
+        _key = key;
+        _val = val;
+    }
+
+    public AbstractEntry(final Map.Entry e) {
+        _key = e.getKey();
+        _val = e.getValue();
+    }
+
+    /**
+     * Return "key=val" string
+     */
+    public String toString() {
+        return _key + "=" + _val;
+    }
+
+    /**
+     * Return key
+     */
+    public TypeK getKey() {
+        return _key;
+    }
+
+    /**
+     * Return val
+     */
+    public TypeV getValue() {
+        return _val;
+    }
+
+    /**
+     * Equal if the underlying key & value are equal
+     */
+    public boolean equals(final Object o) {
+        if (!(o instanceof Map.Entry))
+            return false;
+        final Map.Entry e = (Map.Entry) o;
+        return eq(_key, e.getKey()) && eq(_val, e.getValue());
+    }
+
+    /**
+     * Compute "key.hashCode() ^ val.hashCode()"
+     */
+    public int hashCode() {
+        return ((_key == null) ? 0 : _key.hashCode()) ^ ((_val == null) ? 0 : _val.hashCode());
+    }
+
+    private static boolean eq(final Object o1, final Object o2) {
+        return (o1 == null ? o2 == null : o1.equals(o2));
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentAutoTable.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentAutoTable.java
new file mode 100644
index 0000000..c920a79
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentAutoTable.java
@@ -0,0 +1,290 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.collection;
+
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import sun.misc.Unsafe;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * An auto-resizing table of {@code longs}, supporting low-contention CAS
+ * operations.  Updates are done with CAS's to no particular table element.
+ * The intent is to support highly scalable counters, r/w locks, and other
+ * structures where the updates are associative, loss-free (no-brainer), and
+ * otherwise happen at such a high volume that the cache contention for
+ * CAS'ing a single word is unacceptable.
+ *
+ * @author Cliff Click
+ * @since 1.5
+ *
+ * Forked from JCTools.
+ */
+public class ConcurrentAutoTable implements Serializable {
+
+    private static final long serialVersionUID = -754466836461919739L;
+
+    private static Unsafe     unsafe           = UnsafeUtil.getUnsafeAccessor().getUnsafe();
+
+    // --- public interface ---
+
+    /**
+     * Add the given value to current counter value.  Concurrent updates will
+     * not be lost, but addAndGet or getAndAdd are not implemented because the
+     * total counter value (i.e., {@link #get}) is not atomically updated.
+     * Updates are striped across an array of counters to avoid cache contention
+     * and has been tested with performance scaling linearly up to 768 CPUs.
+     */
+    public void add(long x) {
+        add_if(x);
+    }
+
+    /**
+     * {@link #add} with -1
+     */
+    public void decrement() {
+        add_if(-1L);
+    }
+
+    /**
+     * {@link #add} with +1
+     */
+    public void increment() {
+        add_if(1L);
+    }
+
+    /**
+     * Atomically set the sum of the striped counters to specified value.
+     * Rather more expensive than a simple store, in order to remain atomic.
+     */
+    @SuppressWarnings("StatementWithEmptyBody")
+    public void set(long x) {
+        CAT newcat = new CAT(null, 4, x);
+        // Spin until CAS works
+        while (!CAS_cat(_cat, newcat)) { // NOPMD
+            /* empty */
+        }
+    }
+
+    /**
+     * Current value of the counter.  Since other threads are updating furiously
+     * the value is only approximate, but it includes all counts made by the
+     * current thread.  Requires a pass over the internally striped counters.
+     */
+    public long get() {
+        return _cat.sum();
+    }
+
+    /**
+     * Same as {@link #get}, included for completeness.
+     */
+    public int intValue() {
+        return (int) _cat.sum();
+    }
+
+    /**
+     * Same as {@link #get}, included for completeness.
+     */
+    public long longValue() {
+        return _cat.sum();
+    }
+
+    /**
+     * A cheaper {@link #get}.  Updated only once/millisecond, but as fast as a
+     * simple load instruction when not updating.
+     */
+    public long estimate_get() {
+        return _cat.estimate_sum();
+    }
+
+    /**
+     * Return the counter's {@code long} value converted to a string.
+     */
+    public String toString() {
+        return _cat.toString();
+    }
+
+    /**
+     * A more verbose print than {@link #toString}, showing internal structure.
+     * Useful for debugging.
+     */
+    public void print() {
+        _cat.print();
+    }
+
+    /**
+     * Return the internal counter striping factor.  Useful for diagnosing
+     * performance problems.
+     */
+    public int internal_size() {
+        return _cat._t.length;
+    }
+
+    // Only add 'x' to some slot in table, hinted at by 'hash'.  The sum can
+    // overflow.  Value is CAS'd so no counts are lost.  The CAS is retried until
+    // it succeeds.  Returned value is the old value.
+    private long add_if(long x) {
+        return _cat.add_if(x, hash(), this);
+    }
+
+    // The underlying array of concurrently updated long counters
+    private volatile CAT                                                       _cat        = new CAT(
+                                                                                               null,
+                                                                                               16/*Start Small, Think Big!*/,
+                                                                                               0L);
+    private static final AtomicReferenceFieldUpdater _catUpdater = AtomicReferenceFieldUpdater
+                                                                                               .newUpdater(
+                                                                                                   ConcurrentAutoTable.class,
+                                                                                                   CAT.class, "_cat");
+
+    private boolean CAS_cat(CAT oldcat, CAT newcat) {
+        return _catUpdater.compareAndSet(this, oldcat, newcat);
+    }
+
+    // Hash spreader
+    private static int hash() {
+        //int h = (int)Thread.currentThread().getId();
+        int h = System.identityHashCode(Thread.currentThread());
+        return h << 3; // Pad out cache lines.  The goal is to avoid cache-line contention
+    }
+
+    // --- CAT -----------------------------------------------------------------
+    private static class CAT implements Serializable {
+
+        private static final long serialVersionUID = 7664209842256114771L;
+
+        // Unsafe crud: get a function which will CAS arrays
+        private static final int  _Lbase           = unsafe.arrayBaseOffset(long[].class);
+        private static final int  _Lscale          = unsafe.arrayIndexScale(long[].class);
+
+        private static long rawIndex(long[] ary, int i) {
+            assert i >= 0 && i < ary.length;
+            return _Lbase + ((long) i * _Lscale);
+        }
+
+        private static boolean CAS(long[] A, int idx, long old, long nnn) {
+            return unsafe.compareAndSwapLong(A, rawIndex(A, idx), old, nnn);
+        }
+
+        //volatile long _resizers;    // count of threads attempting a resize
+        //static private final AtomicLongFieldUpdater _resizerUpdater =
+        //  AtomicLongFieldUpdater.newUpdater(CAT.class, "_resizers");
+
+        private final CAT        _next;
+        private volatile long    _fuzzy_sum_cache;
+        private volatile long    _fuzzy_time;
+        private static final int MAX_SPIN = 1;
+        private final long[]     _t;              // Power-of-2 array of longs
+
+        CAT(CAT next, int sz, long init) {
+            _next = next;
+            _t = new long[sz];
+            _t[0] = init;
+        }
+
+        // Only add 'x' to some slot in table, hinted at by 'hash'.  The sum can
+        // overflow.  Value is CAS'd so no counts are lost.  The CAS is attempted
+        // ONCE.
+        @SuppressWarnings("StatementWithEmptyBody")
+        public long add_if(long x, int hash, ConcurrentAutoTable master) {
+            final long[] t = _t;
+            final int idx = hash & (t.length - 1);
+            // Peel loop; try once fast
+            long old = t[idx];
+            final boolean ok = CAS(t, idx, old, old + x);
+            if (ok)
+                return old; // Got it
+            // Try harder
+            int cnt = 0;
+            while (true) {
+                old = t[idx];
+                if (CAS(t, idx, old, old + x))
+                    break; // Got it!
+                cnt++;
+            }
+            if (cnt < MAX_SPIN)
+                return old; // Allowable spin loop count
+            if (t.length >= 1024 * 1024)
+                return old; // too big already
+
+            // Too much contention; double array size in an effort to reduce contention
+            //long r = _resizers;
+            //final int newbytes = (t.length<<1)<<3/*word to bytes*/;
+            //while( !_resizerUpdater.compareAndSet(this,r,r+newbytes) )
+            //  r = _resizers;
+            //r += newbytes;
+            if (master._cat != this)
+                return old; // Already doubled, don't bother
+            //if( (r>>17) != 0 ) {      // Already too much allocation attempts?
+            //  // We could use a wait with timeout, so we'll wakeup as soon as the new
+            //  // table is ready, or after the timeout in any case.  Annoyingly, this
+            //  // breaks the non-blocking property - so for now we just briefly sleep.
+            //  //synchronized( this ) { wait(8*megs); }         // Timeout - we always wakeup
+            //  try { Thread.sleep(r>>17); } catch( InterruptedException e ) { }
+            //  if( master._cat != this ) return old;
+            //}
+
+            CAT newcat = new CAT(this, t.length * 2, 0);
+            // Take 1 stab at updating the CAT with the new larger size.  If this
+            // fails, we assume some other thread already expanded the CAT - so we
+            // do not need to retry until it succeeds.
+            while (master._cat == this && !master.CAS_cat(this, newcat)) { // NOPMD
+                /* empty */
+            }
+            return old;
+        }
+
+        // Return the current sum of all things in the table.  Writers can be
+        // updating the table furiously, so the sum is only locally accurate.
+        @SuppressWarnings("UnnecessaryLocalVariable")
+        public long sum() {
+            long sum = _next == null ? 0 : _next.sum(); // Recursively get cached sum
+            final long[] t = _t;
+            for (long cnt : t)
+                sum += cnt;
+            return sum;
+        }
+
+        // Fast fuzzy version.  Used a cached value until it gets old, then re-up
+        // the cache.
+        public long estimate_sum() {
+            // For short tables, just do the work
+            if (_t.length <= 64)
+                return sum();
+            // For bigger tables, periodically freshen a cached value
+            long millis = System.currentTimeMillis();
+            if (_fuzzy_time != millis) { // Time marches on?
+                _fuzzy_sum_cache = sum(); // Get sum the hard way
+                _fuzzy_time = millis; // Indicate freshness of cached value
+            }
+            return _fuzzy_sum_cache; // Return cached sum
+        }
+
+        public String toString() {
+            return Long.toString(sum());
+        }
+
+        public void print() {
+            long[] t = _t;
+            System.out.print("[" + t[0]); // NOPMD
+            for (int i = 1; i < t.length; i++)
+                System.out.print("," + t[i]); // NOPMD
+            System.out.print("]"); // NOPMD
+            if (_next != null)
+                _next.print();
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentSet.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentSet.java
new file mode 100644
index 0000000..b4fd191
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/ConcurrentSet.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.collection;
+
+import java.io.Serializable;
+import java.util.AbstractSet;
+import java.util.Iterator;
+import java.util.concurrent.ConcurrentMap;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ConcurrentSet extends AbstractSet implements Serializable {
+
+    private static final long               serialVersionUID = -6761513279741915432L;
+
+    private final ConcurrentMap map;
+
+    /**
+     * Creates a new instance which wraps the specified {@code map}.
+     */
+    public ConcurrentSet() {
+        map = Maps.newConcurrentMap();
+    }
+
+    @Override
+    public int size() {
+        return map.size();
+    }
+
+    @SuppressWarnings("SuspiciousMethodCalls")
+    @Override
+    public boolean contains(Object o) {
+        return map.containsKey(o);
+    }
+
+    @Override
+    public boolean add(E o) {
+        return map.putIfAbsent(o, Boolean.TRUE) == null;
+    }
+
+    @Override
+    public boolean remove(Object o) {
+        return map.remove(o) != null;
+    }
+
+    @Override
+    public void clear() {
+        map.clear();
+    }
+
+    @Override
+    public Iterator iterator() {
+        return map.keySet().iterator();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMap.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMap.java
new file mode 100644
index 0000000..2f500ae
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMap.java
@@ -0,0 +1,1678 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.collection;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.util.AbstractCollection;
+import java.util.AbstractMap;
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import sun.misc.Unsafe;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * A lock-free alternate implementation of {@link java.util.concurrent.ConcurrentHashMap}
+ * with better scaling properties and generally lower costs to mutate the Map.
+ * It provides identical correctness properties as ConcurrentHashMap.  All
+ * operations are non-blocking and multi-thread safe, including all update
+ * operations.  {@link NonBlockingHashMap} scales substantially better than
+ * {@link java.util.concurrent.ConcurrentHashMap} for high update rates, even with a
+ * large concurrency factor.  Scaling is linear up to 768 CPUs on a 768-CPU
+ * Azul box, even with 100% updates or 100% reads or any fraction in-between.
+ * Linear scaling up to all cpus has been observed on a 32-way Sun US2 box,
+ * 32-way Sun Niagara box, 8-way Intel box and a 4-way Power box.
+ * 
    
+ * This class obeys the same functional specification as {@link
+ * Hashtable}, and includes versions of methods corresponding to
+ * each method of Hashtable. However, even though all operations are
+ * thread-safe, operations do not entail locking and there is
+ * not any support for locking the entire table in a way that
+ * prevents all access.  This class is fully interoperable with
+ * Hashtable in programs that rely on its thread safety but not on
+ * its synchronization details.
+ * 
    
+ * 
     Operations (including put) generally do not block, so may
+ * overlap with other update operations (including other puts and
+ * removes).  Retrievals reflect the results of the most recently
+ * completed update operations holding upon their onset.  For
+ * aggregate operations such as putAll, concurrent retrievals may
+ * reflect insertion or removal of only some entries.  Similarly, Iterators
+ * and Enumerations return elements reflecting the state of the hash table at
+ * some point at or since the creation of the iterator/enumeration.  They do
+ * not throw {@link ConcurrentModificationException}.  However,
+ * iterators are designed to be used by only one thread at a time.
+ * 
    
+ * 
     Very full tables, or tables with high reprobe rates may trigger an
+ * internal resize operation to move into a larger table.  Resizing is not
+ * terribly expensive, but it is not free either; during resize operations
+ * table throughput may drop somewhat.  All threads that visit the table
+ * during a resize will 'help' the resizing but will still be allowed to
+ * complete their operation before the resize is finished (i.e., a simple
+ * 'get' operation on a million-entry table undergoing resizing will not need
+ * to block until the entire million entries are copied).
+ * 
    
+ * 
    This class and its views and iterators implement all of the
+ * optional methods of the {@link Map} and {@link Iterator}
+ * interfaces.
+ * 
    
+ * 
     Like {@link Hashtable} but unlike {@link HashMap}, this class
+ * does not allow null to be used as a key or value.
+ *
+ * @param  the type of keys maintained by this map
+ * @param  the type of mapped values
+ * @author Cliff Click
+ * @since 1.5
+ *
+ * Forked from JCTools.
+ */
+public class NonBlockingHashMap extends AbstractMap implements ConcurrentMap,
+                                                                               Cloneable, Serializable {
+
+    private static final long serialVersionUID = 1234123412341234123L;
+
+    private static Unsafe     unsafe           = UnsafeUtil.getUnsafeAccessor().getUnsafe();
+
+    private static final int  REPROBE_LIMIT    = 10;                                          // Too many reprobes then force a table-resize
+
+    // --- Bits to allow Unsafe access to arrays
+    private static final int  _Obase           = unsafe.arrayBaseOffset(Object[].class);
+    private static final int  _Oscale          = unsafe.arrayIndexScale(Object[].class);
+    private static final int  _Olog            = _Oscale == 4 ? 2 : (_Oscale == 8 ? 3 : 9999);
+
+    private static long rawIndex(final Object[] ary, final int idx) {
+        assert idx >= 0 && idx < ary.length;
+        // Note the long-math requirement, to handle arrays of more than 2^31 bytes
+        // - or 2^28 - or about 268M - 8-byte pointer elements.
+        return _Obase + ((long) idx << _Olog);
+    }
+
+    // --- Setup to use Unsafe
+    private static final long _kvs_offset;
+
+    static { // 
+        Field f;
+        try {
+            f = NonBlockingHashMap.class.getDeclaredField("_kvs");
+        } catch (NoSuchFieldException e) {
+            throw new RuntimeException(e);
+        }
+        _kvs_offset = unsafe.objectFieldOffset(f);
+    }
+
+    private boolean CAS_kvs(final Object[] oldkvs, final Object[] newkvs) {
+        return unsafe.compareAndSwapObject(this, _kvs_offset, oldkvs, newkvs);
+    }
+
+    // --- Adding a 'prime' bit onto Values via wrapping with a junk wrapper class
+    private static final class Prime {
+        final Object _V;
+
+        Prime(Object V) {
+            _V = V;
+        }
+
+        static Object unbox(Object V) {
+            return V instanceof Prime ? ((Prime) V)._V : V;
+        }
+    }
+
+    // --- hash ----------------------------------------------------------------
+    // Helper function to spread lousy hashCodes
+    private static int hash(final Object key) {
+        int h = key.hashCode(); // The real hashCode call
+        h ^= (h >>> 20) ^ (h >>> 12);
+        h ^= (h >>> 7) ^ (h >>> 4);
+        return h;
+    }
+
+    // --- The Hash Table --------------------
+    // Slot 0 is always used for a 'CHM' entry below to hold the interesting
+    // bits of the hash table.  Slot 1 holds full hashes as an array of ints.
+    // Slots {2,3}, {4,5}, etc hold {Key,Value} pairs.  The entire hash table
+    // can be atomically replaced by CASing the _kvs field.
+    //
+    // Why is CHM buried inside the _kvs Object array, instead of the other way
+    // around?  The CHM info is used during resize events and updates, but not
+    // during standard 'get' operations.  I assume 'get' is much more frequent
+    // than 'put'.  'get' can skip the extra indirection of skipping through the
+    // CHM to reach the _kvs array.
+    private transient Object[] _kvs;
+
+    private static CHM chm(Object[] kvs) {
+        return (CHM) kvs[0];
+    }
+
+    private static int[] hashes(Object[] kvs) {
+        return (int[]) kvs[1];
+    }
+
+    // Number of K,V pairs in the table
+    private static int len(Object[] kvs) {
+        return (kvs.length - 2) >> 1;
+    }
+
+    // Time since last resize
+    private transient long      _last_resize_milli;
+
+    // --- Minimum table size ----------------
+    // Pick size 8 K/V pairs, which turns into (8*2+2)*4+12 = 84 bytes on a
+    // standard 32-bit HotSpot, and (8*2+2)*8+12 = 156 bytes on 64-bit Azul.
+    private static final int    MIN_SIZE_LOG = 3;                   //
+    private static final int    MIN_SIZE     = (1 << MIN_SIZE_LOG); // Must be power of 2
+
+    // --- Sentinels -------------------------
+    // No-Match-Old - putIfMatch does updates only if it matches the old value,
+    // and NO_MATCH_OLD basically counts as a wildcard match.
+    private static final Object NO_MATCH_OLD = new Object();        // Sentinel
+    // Match-Any-not-null - putIfMatch does updates only if it find a real old
+    // value.
+    private static final Object MATCH_ANY    = new Object();        // Sentinel
+    // This K/V pair has been deleted (but the Key slot is forever claimed).
+    // The same Key can be reinserted with a new value later.
+    public static final Object  TOMBSTONE    = new Object();
+    // Prime'd or box'd version of TOMBSTONE.  This K/V pair was deleted, then a
+    // table resize started.  The K/V pair has been marked so that no new
+    // updates can happen to the old table (and since the K/V pair was deleted
+    // nothing was copied to the new table).
+    private static final Prime  TOMBPRIME    = new Prime(TOMBSTONE);
+
+    // --- key,val -------------------------------------------------------------
+    // Access K,V for a given idx
+    //
+    // Note that these are static, so that the caller is forced to read the _kvs
+    // field only once, and share that read across all key/val calls - lest the
+    // _kvs field move out from under us and back-to-back key & val calls refer
+    // to different _kvs arrays.
+    private static Object key(Object[] kvs, int idx) {
+        return kvs[(idx << 1) + 2];
+    }
+
+    private static Object val(Object[] kvs, int idx) {
+        return kvs[(idx << 1) + 3];
+    }
+
+    @SuppressWarnings("SameParameterValue")
+    private static boolean CAS_key(Object[] kvs, int idx, Object old, Object key) {
+        return unsafe.compareAndSwapObject(kvs, rawIndex(kvs, (idx << 1) + 2), old, key);
+    }
+
+    private static boolean CAS_val(Object[] kvs, int idx, Object old, Object val) {
+        return unsafe.compareAndSwapObject(kvs, rawIndex(kvs, (idx << 1) + 3), old, val);
+    }
+
+    // --- dump ----------------------------------------------------------------
+
+    /**
+     * Verbose printout of table internals, useful for debugging.
+     */
+    public final void print() {
+        System.out.println("========="); // NOPMD
+        print2(_kvs);
+        System.out.println("========="); // NOPMD
+    }
+
+    // print the entire state of the table
+    @SuppressWarnings("unused")
+    private void print(Object[] kvs) {
+        for (int i = 0; i < len(kvs); i++) {
+            Object K = key(kvs, i);
+            if (K != null) {
+                String KS = (K == TOMBSTONE) ? "XXX" : K.toString();
+                Object V = val(kvs, i);
+                Object U = Prime.unbox(V);
+                String p = (V == U) ? "" : "prime_";
+                String US = (U == TOMBSTONE) ? "tombstone" : U.toString();
+                System.out.println("" + i + " (" + KS + "," + p + US + ")"); // NOPMD
+            }
+        }
+        Object[] newkvs = chm(kvs)._newkvs; // New table, if any
+        if (newkvs != null) {
+            System.out.println("----"); // NOPMD
+            print(newkvs);
+        }
+    }
+
+    // print only the live values, broken down by the table they are in
+    private void print2(Object[] kvs) {
+        for (int i = 0; i < len(kvs); i++) {
+            Object key = key(kvs, i);
+            Object val = val(kvs, i);
+            Object U = Prime.unbox(val);
+            if (key != null && key != TOMBSTONE && // key is sane
+                val != null && U != TOMBSTONE) { // val is sane
+                String p = (val == U) ? "" : "prime_";
+                System.out.println("" + i + " (" + key + "," + p + val + ")"); // NOPMD
+            }
+        }
+        Object[] newkvs = chm(kvs)._newkvs; // New table, if any
+        if (newkvs != null) {
+            System.out.println("----"); // NOPMD
+            print2(newkvs);
+        }
+    }
+
+    // Count of reprobes
+    private transient ConcurrentAutoTable _reprobes = new ConcurrentAutoTable();
+
+    /**
+     * Get and clear the current count of reprobes.  Reprobes happen on key
+     * collisions, and a high reprobe rate may indicate a poor hash function or
+     * weaknesses in the table resizing function.
+     *
+     * @return the count of reprobes since the last call to reprobes
+     * or since the table was created.
+     */
+    public long reprobes() {
+        long r = _reprobes.get();
+        _reprobes = new ConcurrentAutoTable();
+        return r;
+    }
+
+    // --- reprobe_limit -----------------------------------------------------
+    // Heuristic to decide if we have reprobed toooo many times.  Running over
+    // the reprobe limit on a 'get' call acts as a 'miss'; on a 'put' call it
+    // can trigger a table resize.  Several places must have exact agreement on
+    // what the reprobe_limit is, so we share it here.
+    private static int reprobe_limit(int len) {
+        return REPROBE_LIMIT + (len >> 4);
+    }
+
+    // --- NonBlockingHashMap --------------------------------------------------
+    // Constructors
+
+    /**
+     * Create a new NonBlockingHashMap with default minimum size (currently set
+     * to 8 K/V pairs or roughly 84 bytes on a standard 32-bit JVM).
+     */
+    public NonBlockingHashMap() {
+        this(MIN_SIZE);
+    }
+
+    /**
+     * Create a new NonBlockingHashMap with initial room for the given number of
+     * elements, thus avoiding internal resizing operations to reach an
+     * appropriate size.  Large numbers here when used with a small count of
+     * elements will sacrifice space for a small amount of time gained.  The
+     * initial size will be rounded up internally to the next larger power of 2.
+     */
+    public NonBlockingHashMap(final int initial_sz) {
+        initialize(initial_sz);
+    }
+
+    @SuppressWarnings("StatementWithEmptyBody")
+    private void initialize(int initial_sz) {
+        if (initial_sz < 0)
+            throw new IllegalArgumentException();
+        int i; // Convert to next largest power-of-2
+        if (initial_sz > 1024 * 1024)
+            initial_sz = 1024 * 1024;
+        for (i = MIN_SIZE_LOG; (1 << i) < (initial_sz << 2); i++)
+            ;
+        // Double size for K,V pairs, add 1 for CHM and 1 for hashes
+        _kvs = new Object[((1 << i) << 1) + 2];
+        _kvs[0] = new CHM(new ConcurrentAutoTable()); // CHM in slot 0
+        _kvs[1] = new int[1 << i]; // Matching hash entries
+        _last_resize_milli = System.currentTimeMillis();
+    }
+
+    // Version for subclassed readObject calls, to be called after the defaultReadObject
+    protected final void initialize() {
+        initialize(MIN_SIZE);
+    }
+
+    // --- wrappers ------------------------------------------------------------
+
+    /**
+     * Returns the number of key-value mappings in this map.
+     *
+     * @return the number of key-value mappings in this map
+     */
+    @Override
+    public int size() {
+        return chm(_kvs).size();
+    }
+
+    /**
+     * Returns size() == 0.
+     *
+     * @return size() == 0
+     */
+    @Override
+    public boolean isEmpty() {
+        return size() == 0;
+    }
+
+    /**
+     * Tests if the key in the table using the equals method.
+     *
+     * @return true if the key is in the table using the equals method
+     * @throws NullPointerException if the specified key is null
+     */
+    @Override
+    public boolean containsKey(Object key) {
+        return get(key) != null;
+    }
+
+    /**
+     * Legacy method testing if some key maps into the specified value in this
+     * table.  This method is identical in functionality to {@link
+     * #containsValue}, and exists solely to ensure full compatibility with
+     * class {@link Hashtable}, which supported this method prior to
+     * introduction of the Java Collections framework.
+     *
+     * @param val a value to search for
+     * @return true if this map maps one or more keys to the specified value
+     * @throws NullPointerException if the specified value is null
+     */
+    public boolean contains(Object val) {
+        return containsValue(val);
+    }
+
+    /**
+     * Maps the specified key to the specified value in the table.  Neither key
+     * nor value can be null.
+     * 
     The value can be retrieved by calling {@link #get} with a key that is
+     * equal to the original key.
+     *
+     * @param key key with which the specified value is to be associated
+     * @param val value to be associated with the specified key
+     * @return the previous value associated with key, or
+     * null if there was no mapping for key
+     * @throws NullPointerException if the specified key or value is null
+     */
+    @Override
+    public TypeV put(TypeK key, TypeV val) {
+        return putIfMatch(key, val, NO_MATCH_OLD);
+    }
+
+    /**
+     * Atomically, do a {@link #put} if-and-only-if the key is not mapped.
+     * Useful to ensure that only a single mapping for the key exists, even if
+     * many threads are trying to create the mapping in parallel.
+     *
+     * @return the previous value associated with the specified key,
+     * or null if there was no mapping for the key
+     * @throws NullPointerException if the specified key or value is null
+     */
+    public TypeV putIfAbsent(TypeK key, TypeV val) {
+        return putIfMatch(key, val, TOMBSTONE);
+    }
+
+    /**
+     * Removes the key (and its corresponding value) from this map.
+     * This method does nothing if the key is not in the map.
+     *
+     * @return the previous value associated with key, or
+     * null if there was no mapping for key
+     * @throws NullPointerException if the specified key is null
+     */
+    @Override
+    public TypeV remove(Object key) {
+        return putIfMatch(key, TOMBSTONE, NO_MATCH_OLD);
+    }
+
+    /**
+     * Atomically do a {@link #remove(Object)} if-and-only-if the key is mapped
+     * to a value which is equals to the given value.
+     *
+     * @throws NullPointerException if the specified key or value is null
+     */
+    public boolean remove(Object key, Object val) {
+        return putIfMatch(key, TOMBSTONE, val) == val;
+    }
+
+    /**
+     * Atomically do a put(key,val) if-and-only-if the key is
+     * mapped to some value already.
+     *
+     * @throws NullPointerException if the specified key or value is null
+     */
+    public TypeV replace(TypeK key, TypeV val) {
+        return putIfMatch(key, val, MATCH_ANY);
+    }
+
+    /**
+     * Atomically do a put(key,newValue) if-and-only-if the key is
+     * mapped a value which is equals to oldValue.
+     *
+     * @throws NullPointerException if the specified key or value is null
+     */
+    public boolean replace(TypeK key, TypeV oldValue, TypeV newValue) {
+        return putIfMatch(key, newValue, oldValue) == oldValue;
+    }
+
+    // Atomically replace newVal for oldVal, returning the value that existed
+    // there before.  If oldVal is the returned value, then newVal was inserted,
+    // otherwise not.  A null oldVal means the key does not exist (only insert if
+    // missing); a null newVal means to remove the key.
+    @SuppressWarnings("unchecked")
+    public final TypeV putIfMatchAllowNull(Object key, Object newVal, Object oldVal) {
+        if (oldVal == null)
+            oldVal = TOMBSTONE;
+        if (newVal == null)
+            newVal = TOMBSTONE;
+        final TypeV res = (TypeV) putIfMatch(this, _kvs, key, newVal, oldVal);
+        assert !(res instanceof Prime);
+        //assert res != null;
+        return res == TOMBSTONE ? null : res;
+    }
+
+    @SuppressWarnings("unchecked")
+    private TypeV putIfMatch(Object key, Object newVal, Object oldVal) {
+        if (oldVal == null || newVal == null)
+            throw new NullPointerException();
+        final Object res = putIfMatch(this, _kvs, key, newVal, oldVal);
+        assert !(res instanceof Prime);
+        assert res != null;
+        return res == TOMBSTONE ? null : (TypeV) res;
+    }
+
+    /**
+     * Copies all of the mappings from the specified map to this one, replacing
+     * any existing mappings.
+     *
+     * @param m mappings to be stored in this map
+     */
+    @Override
+    public void putAll(Map m) {
+        for (Entry e : m.entrySet())
+            put(e.getKey(), e.getValue());
+    }
+
+    /**
+     * Removes all of the mappings from this map.
+     */
+    @SuppressWarnings("StatementWithEmptyBody")
+    @Override
+    public void clear() { // Smack a new empty table down
+        Object[] newkvs = new NonBlockingHashMap(MIN_SIZE)._kvs;
+        while (!CAS_kvs(_kvs, newkvs)) { // NOPMD
+            // Spin until the clear works
+        }
+    }
+
+    /**
+     * Returns true if this Map maps one or more keys to the specified
+     * value.  Note: This method requires a full internal traversal of the
+     * hash table and is much slower than {@link #containsKey}.
+     *
+     * @param val value whose presence in this map is to be tested
+     * @return true if this map maps one or more keys to the specified value
+     * @throws NullPointerException if the specified value is null
+     */
+    @Override
+    public boolean containsValue(final Object val) {
+        if (val == null)
+            throw new NullPointerException();
+        for (TypeV V : values())
+            if (V == val || V.equals(val))
+                return true;
+        return false;
+    }
+
+    // This function is supposed to do something for Hashtable, and the JCK
+    // tests hang until it gets called... by somebody ... for some reason,
+    // any reason....
+    protected void rehash() {
+    }
+
+    /**
+     * Creates a shallow copy of this hashtable. All the structure of the
+     * hashtable itself is copied, but the keys and values are not cloned.
+     * This is a relatively expensive operation.
+     *
+     * @return a clone of the hashtable.
+     */
+    @SuppressWarnings("unchecked")
+    @Override
+    public Object clone() {
+        try {
+            // Must clone, to get the class right; NBHM might have been
+            // extended so it would be wrong to just make a new NBHM.
+            NonBlockingHashMap t = (NonBlockingHashMap) super.clone();
+            // But I don't have an atomic clone operation - the underlying _kvs
+            // structure is undergoing rapid change.  If I just clone the _kvs
+            // field, the CHM in _kvs[0] won't be in sync.
+            //
+            // Wipe out the cloned array (it was shallow anyways).
+            t.clear();
+            // Now copy sanely
+            for (TypeK K : keySet()) {
+                final TypeV V = get(K); // Do an official 'get'
+                t.put(K, V);
+            }
+            return t;
+        } catch (CloneNotSupportedException e) {
+            // this shouldn't happen, since we are Cloneable
+            throw new InternalError();
+        }
+    }
+
+    /**
+     * Returns a string representation of this map.  The string representation
+     * consists of a list of key-value mappings in the order returned by the
+     * map's entrySet view's iterator, enclosed in braces
+     * ("{}").  Adjacent mappings are separated by the characters
+     * ", " (comma and space).  Each key-value mapping is rendered as
+     * the key followed by an equals sign ("=") followed by the
+     * associated value.  Keys and values are converted to strings as by
+     * {@link String#valueOf(Object)}.
+     *
+     * @return a string representation of this map
+     */
+    @Override
+    public String toString() {
+        Iterator> i = entrySet().iterator();
+        if (!i.hasNext())
+            return "{}";
+
+        StringBuilder sb = new StringBuilder();
+        sb.append('{');
+        for (;;) {
+            Entry e = i.next();
+            TypeK key = e.getKey();
+            TypeV value = e.getValue();
+            sb.append(key == this ? "(this Map)" : key);
+            sb.append('=');
+            sb.append(value == this ? "(this Map)" : value);
+            if (!i.hasNext())
+                return sb.append('}').toString();
+            sb.append(", ");
+        }
+    }
+
+    // --- keyeq ---------------------------------------------------------------
+    // Check for key equality.  Try direct pointer compare first, then see if
+    // the hashes are unequal (fast negative test) and finally do the full-on
+    // 'equals' v-call.
+    private static boolean keyeq(Object K, Object key, int[] hashes, int hash, int fullhash) {
+        return K == key || // Either keys match exactly OR
+               // hash exists and matches?  hash can be zero during the install of a
+               // new key/value pair.
+               ((hashes[hash] == 0 || hashes[hash] == fullhash) &&
+               // Do not call the users' "equals()" call with a Tombstone, as this can
+               // surprise poorly written "equals()" calls that throw exceptions
+               // instead of simply returning false.
+                K != TOMBSTONE && // Do not call users' equals call with a Tombstone
+               // Do the match the hard way - with the users' key being the loop-
+               // invariant "this" pointer.  I could have flipped the order of
+               // operands (since equals is commutative), but I'm making mega-morphic
+               // v-calls in a reprobing loop and nailing down the 'this' argument
+               // gives both the JIT and the hardware a chance to prefetch the call target.
+               key.equals(K)); // Finally do the hard match
+    }
+
+    // --- get -----------------------------------------------------------------
+
+    /**
+     * Returns the value to which the specified key is mapped, or {@code null}
+     * if this map contains no mapping for the key.
+     * 
    More formally, if this map contains a mapping from a key {@code k} to
+     * a value {@code v} such that {@code key.equals(k)}, then this method
+     * returns {@code v}; otherwise it returns {@code null}.  (There can be at
+     * most one such mapping.)
+     *
+     * @throws NullPointerException if the specified key is null
+     */
+    // Never returns a Prime nor a Tombstone.
+    @SuppressWarnings("unchecked")
+    @Override
+    public TypeV get(Object key) {
+        final Object V = get_impl(this, _kvs, key);
+        assert !(V instanceof Prime); // Never return a Prime
+        assert V != TOMBSTONE;
+        return (TypeV) V;
+    }
+
+    private static Object get_impl(final NonBlockingHashMap topmap, final Object[] kvs, final Object key) {
+        final int fullhash = hash(key); // throws NullPointerException if key is null
+        final int len = len(kvs); // Count of key/value pairs, reads kvs.length
+        final CHM chm = chm(kvs); // The CHM, for a volatile read below; reads slot 0 of kvs
+        final int[] hashes = hashes(kvs); // The memoized hashes; reads slot 1 of kvs
+
+        int idx = fullhash & (len - 1); // First key hash
+
+        // Main spin/reprobe loop, looking for a Key hit
+        int reprobe_cnt = 0;
+        while (true) {
+            // Probe table.  Each read of 'val' probably misses in cache in a big
+            // table; hopefully the read of 'key' then hits in cache.
+            final Object K = key(kvs, idx); // Get key   before volatile read, could be null
+            final Object V = val(kvs, idx); // Get value before volatile read, could be null or Tombstone or Prime
+            if (K == null)
+                return null; // A clear miss
+
+            // We need a volatile-read here to preserve happens-before semantics on
+            // newly inserted Keys.  If the Key body was written just before inserting
+            // into the table a Key-compare here might read the uninitialized Key body.
+            // Annoyingly this means we have to volatile-read before EACH key compare.
+            // .
+            // We also need a volatile-read between reading a newly inserted Value
+            // and returning the Value (so the user might end up reading the stale
+            // Value contents).  Same problem as with keys - and the one volatile
+            // read covers both.
+            final Object[] newkvs = chm._newkvs; // VOLATILE READ before key compare
+
+            // Key-compare
+            if (keyeq(K, key, hashes, idx, fullhash)) {
+                // Key hit!  Check for no table-copy-in-progress
+                if (!(V instanceof Prime)) // No copy?
+                    return (V == TOMBSTONE) ? null : V; // Return the value
+                // Key hit - but slot is (possibly partially) copied to the new table.
+                // Finish the copy & retry in the new table.
+                return get_impl(topmap, chm.copy_slot_and_check(topmap, kvs, idx, key), key); // Retry in the new table
+            }
+            // get and put must have the same key lookup logic!  But only 'put'
+            // needs to force a table-resize for a too-long key-reprobe sequence.
+            // Check for too-many-reprobes on get - and flip to the new table.
+            if (++reprobe_cnt >= reprobe_limit(len) || // too many probes
+                K == TOMBSTONE) // found a TOMBSTONE key, means no more keys in this table
+                return newkvs == null ? null : get_impl(topmap, topmap.help_copy(newkvs), key); // Retry in the new table
+
+            idx = (idx + 1) & (len - 1); // Reprobe by 1!  (could now prefetch)
+        }
+    }
+
+    // --- getk -----------------------------------------------------------------
+
+    /**
+     * Returns the Key to which the specified key is mapped, or {@code null}
+     * if this map contains no mapping for the key.
+     *
+     * @throws NullPointerException if the specified key is null
+     */
+    // Never returns a Prime nor a Tombstone.
+    @SuppressWarnings("unchecked")
+    public TypeK getk(TypeK key) {
+        return (TypeK) getk_impl(this, _kvs, key);
+    }
+
+    private static Object getk_impl(final NonBlockingHashMap topmap, final Object[] kvs, final Object key) {
+        final int fullhash = hash(key); // throws NullPointerException if key is null
+        final int len = len(kvs); // Count of key/value pairs, reads kvs.length
+        final CHM chm = chm(kvs); // The CHM, for a volatile read below; reads slot 0 of kvs
+        final int[] hashes = hashes(kvs); // The memoized hashes; reads slot 1 of kvs
+
+        int idx = fullhash & (len - 1); // First key hash
+
+        // Main spin/reprobe loop, looking for a Key hit
+        int reprobe_cnt = 0;
+        while (true) {
+            // Probe table.
+            final Object K = key(kvs, idx); // Get key before volatile read, could be null
+            if (K == null)
+                return null; // A clear miss
+
+            // We need a volatile-read here to preserve happens-before semantics on
+            // newly inserted Keys.  If the Key body was written just before inserting
+            // into the table a Key-compare here might read the uninitialized Key body.
+            // Annoyingly this means we have to volatile-read before EACH key compare.
+            // .
+            // We also need a volatile-read between reading a newly inserted Value
+            // and returning the Value (so the user might end up reading the stale
+            // Value contents).  Same problem as with keys - and the one volatile
+            // read covers both.
+            final Object[] newkvs = chm._newkvs; // VOLATILE READ before key compare
+
+            // Key-compare
+            if (keyeq(K, key, hashes, idx, fullhash))
+                return K; // Return existing Key!
+
+            // get and put must have the same key lookup logic!  But only 'put'
+            // needs to force a table-resize for a too-long key-reprobe sequence.
+            // Check for too-many-reprobes on get - and flip to the new table.
+            if (++reprobe_cnt >= reprobe_limit(len) || // too many probes
+                K == TOMBSTONE) { // found a TOMBSTONE key, means no more keys in this table
+                return newkvs == null ? null : getk_impl(topmap, topmap.help_copy(newkvs), key); // Retry in the new table
+            }
+
+            idx = (idx + 1) & (len - 1); // Reprobe by 1!  (could now prefetch)
+        }
+    }
+
+    // --- putIfMatch ---------------------------------------------------------
+    // Put, Remove, PutIfAbsent, etc.  Return the old value.  If the returned
+    // value is equal to expVal (or expVal is NO_MATCH_OLD) then the put can be
+    // assumed to work (although might have been immediately overwritten).  Only
+    // the path through copy_slot passes in an expected value of null, and
+    // putIfMatch only returns a null if passed in an expected null.
+    static volatile int DUMMY_VOLATILE;
+
+    @SuppressWarnings("unused")
+    private static Object putIfMatch(final NonBlockingHashMap topmap, final Object[] kvs, final Object key,
+                                     final Object putval, final Object expVal) {
+        assert putval != null;
+        assert !(putval instanceof Prime);
+        assert !(expVal instanceof Prime);
+        final int fullhash = hash(key); // throws NullPointerException if key null
+        final int len = len(kvs); // Count of key/value pairs, reads kvs.length
+        final CHM chm = chm(kvs); // Reads kvs[0]
+        final int[] hashes = hashes(kvs); // Reads kvs[1], read before kvs[0]
+        int idx = fullhash & (len - 1);
+
+        // ---
+        // Key-Claim stanza: spin till we can claim a Key (or force a resizing).
+        int reprobe_cnt = 0;
+        Object K, V;
+        Object[] newkvs = null;
+        while (true) { // Spin till we get a Key slot
+            V = val(kvs, idx); // Get old value (before volatile read below!)
+            K = key(kvs, idx); // Get current key
+            if (K == null) { // Slot is free?
+                // Found an empty Key slot - which means this Key has never been in
+                // this table.  No need to put a Tombstone - the Key is not here!
+                if (putval == TOMBSTONE)
+                    return putval; // Not-now & never-been in this table
+                if (expVal == MATCH_ANY)
+                    return null; // Will not match, even after K inserts
+                // Claim the null key-slot
+                if (CAS_key(kvs, idx, null, key)) { // Claim slot for Key
+                    chm._slots.add(1); // Raise key-slots-used count
+                    hashes[idx] = fullhash; // Memoize fullhash
+                    break; // Got it!
+                }
+                // CAS to claim the key-slot failed.
+                //
+                // This re-read of the Key points out an annoying short-coming of Java
+                // CAS.  Most hardware CAS's report back the existing value - so that
+                // if you fail you have a *witness* - the value which caused the CAS to
+                // fail.  The Java API turns this into a boolean destroying the
+                // witness.  Re-reading does not recover the witness because another
+                // thread can write over the memory after the CAS.  Hence we can be in
+                // the unfortunate situation of having a CAS fail *for cause* but
+                // having that cause removed by a later store.  This turns a
+                // non-spurious-failure CAS (such as Azul has) into one that can
+                // apparently spuriously fail - and we avoid apparent spurious failure
+                // by not allowing Keys to ever change.
+
+                // Volatile read, to force loads of K to retry despite JIT, otherwise
+                // it is legal to e.g. haul the load of "K = key(kvs,idx);" outside of
+                // this loop (since failed CAS ops have no memory ordering semantics).
+                int dummy = DUMMY_VOLATILE;
+                continue;
+            }
+            // Key slot was not null, there exists a Key here
+
+            // We need a volatile-read here to preserve happens-before semantics on
+            // newly inserted Keys.  If the Key body was written just before inserting
+            // into the table a Key-compare here might read the uninitialized Key body.
+            // Annoyingly this means we have to volatile-read before EACH key compare.
+            newkvs = chm._newkvs; // VOLATILE READ before key compare
+
+            if (keyeq(K, key, hashes, idx, fullhash))
+                break; // Got it!
+
+            // get and put must have the same key lookup logic!  Lest 'get' give
+            // up looking too soon.
+            //topmap._reprobes.add(1);
+            if (++reprobe_cnt >= reprobe_limit(len) || // too many probes or
+                K == TOMBSTONE) { // found a TOMBSTONE key, means no more keys
+                // We simply must have a new table to do a 'put'.  At this point a
+                // 'get' will also go to the new table (if any).  We do not need
+                // to claim a key slot (indeed, we cannot find a free one to claim!).
+                newkvs = chm.resize(topmap, kvs);
+                if (expVal != null)
+                    topmap.help_copy(newkvs); // help along an existing copy
+                return putIfMatch(topmap, newkvs, key, putval, expVal);
+            }
+
+            idx = (idx + 1) & (len - 1); // Reprobe!
+        } // End of spinning till we get a Key slot
+
+        // ---
+        // Found the proper Key slot, now update the matching Value slot.  We
+        // never put a null, so Value slots monotonically move from null to
+        // not-null (deleted Values use Tombstone).  Thus if 'V' is null we
+        // fail this fast cutout and fall into the check for table-full.
+        if (putval == V)
+            return V; // Fast cutout for no-change
+
+        // See if we want to move to a new table (to avoid high average re-probe
+        // counts).  We only check on the initial set of a Value from null to
+        // not-null (i.e., once per key-insert).  Of course we got a 'free' check
+        // of newkvs once per key-compare (not really free, but paid-for by the
+        // time we get here).
+        if (newkvs == null && // New table-copy already spotted?
+            // Once per fresh key-insert check the hard way
+            ((V == null && chm.tableFull(reprobe_cnt, len)) ||
+            // Or we found a Prime, but the JMM allowed reordering such that we
+            // did not spot the new table (very rare race here: the writing
+            // thread did a CAS of _newkvs then a store of a Prime.  This thread
+            // reads the Prime, then reads _newkvs - but the read of Prime was so
+            // delayed (or the read of _newkvs was so accelerated) that they
+            // swapped and we still read a null _newkvs.  The resize call below
+            // will do a CAS on _newkvs forcing the read.
+            V instanceof Prime))
+            newkvs = chm.resize(topmap, kvs); // Force the new table copy to start
+        // See if we are moving to a new table.
+        // If so, copy our slot and retry in the new table.
+        if (newkvs != null)
+            return putIfMatch(topmap, chm.copy_slot_and_check(topmap, kvs, idx, expVal), key, putval, expVal);
+
+        // ---
+        // We are finally prepared to update the existing table
+        assert !(V instanceof Prime);
+
+        // Must match old, and we do not?  Then bail out now.  Note that either V
+        // or expVal might be TOMBSTONE.  Also V can be null, if we've never
+        // inserted a value before.  expVal can be null if we are called from
+        // copy_slot.
+
+        if (expVal != NO_MATCH_OLD && // Do we care about expected-Value at all?
+            V != expVal && // No instant match already?
+            (expVal != MATCH_ANY || V == TOMBSTONE || V == null) && !(V == null && expVal == TOMBSTONE) && // Match on null/TOMBSTONE combo
+            (expVal == null || !expVal.equals(V))) // Expensive equals check at the last
+            return V; // Do not update!
+
+        // Actually change the Value in the Key,Value pair
+        if (CAS_val(kvs, idx, V, putval)) {
+            // CAS succeeded - we did the update!
+            // Both normal put's and table-copy calls putIfMatch, but table-copy
+            // does not (effectively) increase the number of live k/v pairs.
+            if (expVal != null) {
+                // Adjust sizes - a striped counter
+                if ((V == null || V == TOMBSTONE) && putval != TOMBSTONE)
+                    chm._size.add(1);
+                if (!(V == null || V == TOMBSTONE) && putval == TOMBSTONE)
+                    chm._size.add(-1);
+            }
+        } else { // Else CAS failed
+            V = val(kvs, idx); // Get new value
+            // If a Prime'd value got installed, we need to re-run the put on the
+            // new table.  Otherwise we lost the CAS to another racing put.
+            // Simply retry from the start.
+            if (V instanceof Prime)
+                return putIfMatch(topmap, chm.copy_slot_and_check(topmap, kvs, idx, expVal), key, putval, expVal);
+        }
+        // Win or lose the CAS, we are done.  If we won then we know the update
+        // happened as expected.  If we lost, it means "we won but another thread
+        // immediately stomped our update with no chance of a reader reading".
+        return (V == null && expVal != null) ? TOMBSTONE : V;
+    }
+
+    // --- help_copy ---------------------------------------------------------
+    // Help along an existing resize operation.  This is just a fast cut-out
+    // wrapper, to encourage inlining for the fast no-copy-in-progress case.  We
+    // always help the top-most table copy, even if there are nested table
+    // copies in progress.
+    private Object[] help_copy(Object[] helper) {
+        // Read the top-level KVS only once.  We'll try to help this copy along,
+        // even if it gets promoted out from under us (i.e., the copy completes
+        // and another KVS becomes the top-level copy).
+        Object[] topkvs = _kvs;
+        CHM topchm = chm(topkvs);
+        if (topchm._newkvs == null)
+            return helper; // No copy in-progress
+        topchm.help_copy_impl(this, topkvs, false);
+        return helper;
+    }
+
+    // --- CHM -----------------------------------------------------------------
+    // The control structure for the NonBlockingHashMap
+    private static final class CHM {
+        // Size in active K,V pairs
+        private final ConcurrentAutoTable _size;
+
+        public int size() {
+            return (int) _size.get();
+        }
+
+        // ---
+        // These next 2 fields are used in the resizing heuristics, to judge when
+        // it is time to resize or copy the table.  Slots is a count of used-up
+        // key slots, and when it nears a large fraction of the table we probably
+        // end up reprobing too much.  Last-resize-milli is the time since the
+        // last resize; if we are running back-to-back resizes without growing
+        // (because there are only a few live keys but many slots full of dead
+        // keys) then we need a larger table to cut down on the churn.
+
+        // Count of used slots, to tell when table is full of dead unusable slots
+        private final ConcurrentAutoTable _slots;
+
+        @SuppressWarnings("unused")
+        public int slots() {
+            return (int) _slots.get();
+        }
+
+        // ---
+        // New mappings, used during resizing.
+        // The 'new KVs' array - created during a resize operation.  This
+        // represents the new table being copied from the old one.  It's the
+        // volatile variable that is read as we cross from one table to the next,
+        // to get the required memory orderings.  It monotonically transits from
+        // null to set (once).
+        volatile Object[]                                               _newkvs;
+        private static final AtomicReferenceFieldUpdater _newkvsUpdater = AtomicReferenceFieldUpdater
+                                                                                           .newUpdater(CHM.class,
+                                                                                               Object[].class,
+                                                                                               "_newkvs");
+
+        // Set the _next field if we can.
+        boolean CAS_newkvs(Object[] newkvs) {
+            while (_newkvs == null)
+                if (_newkvsUpdater.compareAndSet(this, null, newkvs))
+                    return true;
+            return false;
+        }
+
+        // Sometimes many threads race to create a new very large table.  Only 1
+        // wins the race, but the losers all allocate a junk large table with
+        // hefty allocation costs.  Attempt to control the overkill here by
+        // throttling attempts to create a new table.  I cannot really block here
+        // (lest I lose the non-blocking property) but late-arriving threads can
+        // give the initial resizing thread a little time to allocate the initial
+        // new table.  The Right Long Term Fix here is to use array-lets and
+        // incrementally create the new very large array.  In C I'd make the array
+        // with malloc (which would mmap under the hood) which would only eat
+        // virtual-address and not real memory - and after Somebody wins then we
+        // could in parallel initialize the array.  Java does not allow
+        // un-initialized array creation (especially of ref arrays!).
+        volatile long                                    _resizers;                       // count of threads attempting an initial resize
+        private static final AtomicLongFieldUpdater _resizerUpdater = AtomicLongFieldUpdater.newUpdater(CHM.class,
+                                                                             "_resizers");
+
+        // ---
+        // Simple constructor
+        CHM(ConcurrentAutoTable size) {
+            _size = size;
+            _slots = new ConcurrentAutoTable();
+        }
+
+        // --- tableFull ---------------------------------------------------------
+        // Heuristic to decide if this table is too full, and we should start a
+        // new table.  Note that if a 'get' call has reprobed too many times and
+        // decided the table must be full, then always the estimate_sum must be
+        // high and we must report the table is full.  If we do not, then we might
+        // end up deciding that the table is not full and inserting into the
+        // current table, while a 'get' has decided the same key cannot be in this
+        // table because of too many reprobes.  The invariant is:
+        //   slots.estimate_sum >= max_reprobe_cnt >= reprobe_limit(len)
+        private boolean tableFull(int reprobe_cnt, int len) {
+            return
+            // Do the cheap check first: we allow some number of reprobes always
+            reprobe_cnt >= REPROBE_LIMIT && (reprobe_cnt >= reprobe_limit(len) ||
+            // More expensive check: see if the table is > 1/2 full.
+                    _slots.estimate_get() >= (len >> 1));
+        }
+
+        // --- resize ------------------------------------------------------------
+        // Resizing after too many probes.  "How Big???" heuristics are here.
+        // Callers will (not this routine) will 'help_copy' any in-progress copy.
+        // Since this routine has a fast cutout for copy-already-started, callers
+        // MUST 'help_copy' lest we have a path which forever runs through
+        // 'resize' only to discover a copy-in-progress which never progresses.
+        @SuppressWarnings({ "StatementWithEmptyBody", "unused", "UnusedAssignment" })
+        private Object[] resize(NonBlockingHashMap topmap, Object[] kvs) {
+            assert chm(kvs) == this;
+
+            // Check for resize already in progress, probably triggered by another thread
+            Object[] newkvs = _newkvs; // VOLATILE READ
+            if (newkvs != null) // See if resize is already in progress
+                return newkvs; // Use the new table already
+
+            // No copy in-progress, so start one.  First up: compute new table size.
+            int oldlen = len(kvs); // Old count of K,V pairs allowed
+            int sz = size(); // Get current table count of active K,V pairs
+            int newsz = sz; // First size estimate
+
+            // Heuristic to determine new size.  We expect plenty of dead-slots-with-keys
+            // and we need some decent padding to avoid endless reprobing.
+            if (sz >= (oldlen >> 2)) { // If we are >25% full of keys then...
+                newsz = oldlen << 1; // Double size, so new table will be between 12.5% and 25% full
+                // For tables less than 1M entries, if >50% full of keys then...
+                // For tables more than 1M entries, if >75% full of keys then...
+                if (4L * sz >= ((oldlen >> 20) != 0 ? 3L : 2L) * oldlen)
+                    newsz = oldlen << 2; // Double double size, so new table will be between %12.5 (18.75%) and 25% (25%)
+            }
+            // This heuristic in the next 2 lines leads to a much denser table
+            // with a higher reprobe rate
+            //if( sz >= (oldlen>>1) ) // If we are >50% full of keys then...
+            //  newsz = oldlen<<1;    // Double size
+
+            // Last (re)size operation was very recent?  Then double again; slows
+            // down resize operations for tables subject to a high key churn rate.
+            long tm = System.currentTimeMillis();
+            long q = 0;
+            if (newsz <= oldlen && // New table would shrink or hold steady?
+                (tm <= topmap._last_resize_milli + 10000 || // Recent resize (less than 10 sec ago)
+                (q = _slots.estimate_get()) >= (sz << 1))) // 1/2 of keys are dead?
+                newsz = oldlen << 1; // Double the existing size
+
+            // Do not shrink, ever
+            if (newsz < oldlen)
+                newsz = oldlen;
+
+            // Convert to power-of-2
+            int log2;
+            for (log2 = MIN_SIZE_LOG; (1 << log2) < newsz; log2++)
+                ; // Compute log2 of size
+            long len = ((1L << log2) << 1) + 2;
+            // prevent integer overflow - limit of 2^31 elements in a Java array
+            // so here, 2^30 + 2 is the largest number of elements in the hash table
+            if ((int) len != len) {
+                log2 = 30;
+                len = (1L << log2) + 2;
+                if (sz > ((len >> 2) + (len >> 1)))
+                    throw new RuntimeException("Table is full.");
+            }
+
+            // Now limit the number of threads actually allocating memory to a
+            // handful - lest we have 750 threads all trying to allocate a giant
+            // resized array.
+            long r = _resizers;
+            while (!_resizerUpdater.compareAndSet(this, r, r + 1))
+                r = _resizers;
+            // Size calculation: 2 words (K+V) per table entry, plus a handful.  We
+            // guess at 64-bit pointers; 32-bit pointers screws up the size calc by
+            // 2x but does not screw up the heuristic very much.
+            long megs = ((((1L << log2) << 1) + 8) << 3/*word to bytes*/) >> 20/*megs*/;
+            if (r >= 2 && megs > 0) { // Already 2 guys trying; wait and see
+                newkvs = _newkvs; // Between dorking around, another thread did it
+                if (newkvs != null) // See if resize is already in progress
+                    return newkvs; // Use the new table already
+                // TODO - use a wait with timeout, so we'll wakeup as soon as the new table
+                // is ready, or after the timeout in any case.
+                //synchronized( this ) { wait(8*megs); }         // Timeout - we always wakeup
+                // For now, sleep a tad and see if the 2 guys already trying to make
+                // the table actually get around to making it happen.
+                try {
+                    Thread.sleep(megs);
+                } catch (Exception ignored) {
+                    // ignored
+                }
+            }
+            // Last check, since the 'new' below is expensive and there is a chance
+            // that another thread slipped in a new thread while we ran the heuristic.
+            newkvs = _newkvs;
+            if (newkvs != null) // See if resize is already in progress
+                return newkvs; // Use the new table already
+
+            // Double size for K,V pairs, add 1 for CHM
+            newkvs = new Object[(int) len]; // This can get expensive for big arrays
+            newkvs[0] = new CHM(_size); // CHM in slot 0
+            newkvs[1] = new int[1 << log2]; // hashes in slot 1
+
+            // Another check after the slow allocation
+            if (_newkvs != null) // See if resize is already in progress
+                return _newkvs; // Use the new table already
+
+            // The new table must be CAS'd in so only 1 winner amongst duplicate
+            // racing resizing threads.  Extra CHM's will be GC'd.
+            if (CAS_newkvs(newkvs)) { // NOW a resize-is-in-progress!
+                //notifyAll();            // Wake up any sleepers
+                //long nano = System.nanoTime();
+                //System.out.println(" "+nano+" Resize from "+oldlen+" to "+(1< _copyIdxUpdater  = AtomicLongFieldUpdater.newUpdater(
+                                                                              CHM.class, "_copyIdx");
+
+        // Work-done reporting.  Used to efficiently signal when we can move to
+        // the new table.  From 0 to len(oldkvs) refers to copying from the old
+        // table to the new.
+        volatile long                                    _copyDone        = 0;
+        static private final AtomicLongFieldUpdater _copyDoneUpdater = AtomicLongFieldUpdater.newUpdater(
+                                                                              CHM.class, "_copyDone");
+
+        // --- help_copy_impl ----------------------------------------------------
+        // Help along an existing resize operation.  We hope its the top-level
+        // copy (it was when we started) but this CHM might have been promoted out
+        // of the top position.
+        private void help_copy_impl(NonBlockingHashMap topmap, Object[] oldkvs, boolean copy_all) {
+            assert chm(oldkvs) == this;
+            Object[] newkvs = _newkvs;
+            assert newkvs != null; // Already checked by caller
+            int oldlen = len(oldkvs); // Total amount to copy
+            final int MIN_COPY_WORK = Math.min(oldlen, 1024); // Limit per-thread work
+
+            // ---
+            int panic_start = -1;
+            int copyidx = -9999; // Fool javac to think it's initialized
+            while (_copyDone < oldlen) { // Still needing to copy?
+                // Carve out a chunk of work.  The counter wraps around so every
+                // thread eventually tries to copy every slot repeatedly.
+
+                // We "panic" if we have tried TWICE to copy every slot - and it still
+                // has not happened.  i.e., twice some thread somewhere claimed they
+                // would copy 'slot X' (by bumping _copyIdx) but they never claimed to
+                // have finished (by bumping _copyDone).  Our choices become limited:
+                // we can wait for the work-claimers to finish (and become a blocking
+                // algorithm) or do the copy work ourselves.  Tiny tables with huge
+                // thread counts trying to copy the table often 'panic'.
+                if (panic_start == -1) { // No panic?
+                    copyidx = (int) _copyIdx;
+                    while (!_copyIdxUpdater.compareAndSet(this, copyidx, copyidx + MIN_COPY_WORK))
+                        copyidx = (int) _copyIdx; // Re-read
+                    if (!(copyidx < (oldlen << 1))) // Panic!
+                        panic_start = copyidx; // Record where we started to panic-copy
+                }
+
+                // We now know what to copy.  Try to copy.
+                int workdone = 0;
+                for (int i = 0; i < MIN_COPY_WORK; i++)
+                    if (copy_slot(topmap, (copyidx + i) & (oldlen - 1), oldkvs, newkvs)) // Made an oldtable slot go dead?
+                        workdone++; // Yes!
+                if (workdone > 0) // Report work-done occasionally
+                    copy_check_and_promote(topmap, oldkvs, workdone);// See if we can promote
+                //for( int i=0; i 0) {
+                while (!_copyDoneUpdater.compareAndSet(this, copyDone, copyDone + workdone)) {
+                    copyDone = _copyDone; // Reload, retry
+                    assert (copyDone + workdone) <= oldlen;
+                }
+                //if( (10*copyDone/oldlen) != (10*(copyDone+workdone)/oldlen) )
+                //System.out.print(" "+(copyDone+workdone)*100/oldlen+"%"+"_"+(_copyIdx*100/oldlen)+"%");
+            }
+
+            // Check for copy being ALL done, and promote.  Note that we might have
+            // nested in-progress copies and manage to finish a nested copy before
+            // finishing the top-level copy.  We only promote top-level copies.
+            if (copyDone + workdone == oldlen && // Ready to promote this table?
+                topmap._kvs == oldkvs && // Looking at the top-level table?
+                // Attempt to promote
+                topmap.CAS_kvs(oldkvs, _newkvs)) {
+                topmap._last_resize_milli = System.currentTimeMillis(); // Record resize time for next check
+                //long nano = System.nanoTime();
+                //System.out.println(" "+nano+" Promote table to "+len(_newkvs));
+                //if( System.out != null ) System.out.print("]");
+            }
+        }
+
+        // --- copy_slot ---------------------------------------------------------
+        // Copy one K/V pair from oldkvs[i] to newkvs.  Returns true if we can
+        // confirm that the new table guaranteed has a value for this old-table
+        // slot.  We need an accurate confirmed-copy count so that we know when we
+        // can promote (if we promote the new table too soon, other threads may
+        // 'miss' on values not-yet-copied from the old table).  We don't allow
+        // any direct updates on the new table, unless they first happened to the
+        // old table - so that any transition in the new table from null to
+        // not-null must have been from a copy_slot (or other old-table overwrite)
+        // and not from a thread directly writing in the new table.  Thus we can
+        // count null-to-not-null transitions in the new table.
+        private boolean copy_slot(NonBlockingHashMap topmap, int idx, Object[] oldkvs, Object[] newkvs) {
+            // Blindly set the key slot from null to TOMBSTONE, to eagerly stop
+            // fresh put's from inserting new values in the old table when the old
+            // table is mid-resize.  We don't need to act on the results here,
+            // because our correctness stems from box'ing the Value field.  Slamming
+            // the Key field is a minor speed optimization.
+            Object key;
+            while ((key = key(oldkvs, idx)) == null)
+                CAS_key(oldkvs, idx, null, TOMBSTONE);
+
+            // ---
+            // Prevent new values from appearing in the old table.
+            // Box what we see in the old table, to prevent further updates.
+            Object oldval = val(oldkvs, idx); // Read OLD table
+            while (!(oldval instanceof Prime)) {
+                final Prime box = (oldval == null || oldval == TOMBSTONE) ? TOMBPRIME : new Prime(oldval);
+                if (CAS_val(oldkvs, idx, oldval, box)) { // CAS down a box'd version of oldval
+                    // If we made the Value slot hold a TOMBPRIME, then we both
+                    // prevented further updates here but also the (absent)
+                    // oldval is vacuously available in the new table.  We
+                    // return with true here: any thread looking for a value for
+                    // this key can correctly go straight to the new table and
+                    // skip looking in the old table.
+                    if (box == TOMBPRIME)
+                        return true;
+                    // Otherwise we boxed something, but it still needs to be
+                    // copied into the new table.
+                    oldval = box; // Record updated oldval
+                    break; // Break loop; oldval is now boxed by us
+                }
+                oldval = val(oldkvs, idx); // Else try, try again
+            }
+            if (oldval == TOMBPRIME)
+                return false; // Copy already complete here!
+
+            // ---
+            // Copy the value into the new table, but only if we overwrite a null.
+            // If another value is already in the new table, then somebody else
+            // wrote something there and that write is happens-after any value that
+            // appears in the old table.  If putIfMatch does not find a null in the
+            // new table - somebody else should have recorded the null-not_null
+            // transition in this copy.
+            Object old_unboxed = ((Prime) oldval)._V;
+            assert old_unboxed != TOMBSTONE;
+            boolean copied_into_new = (putIfMatch(topmap, newkvs, key, old_unboxed, null) == null);
+
+            // ---
+            // Finally, now that any old value is exposed in the new table, we can
+            // forever hide the old-table value by slapping a TOMBPRIME down.  This
+            // will stop other threads from uselessly attempting to copy this slot
+            // (i.e., it's a speed optimization not a correctness issue).
+            while (oldval != TOMBPRIME && !CAS_val(oldkvs, idx, oldval, TOMBPRIME))
+                oldval = val(oldkvs, idx);
+
+            return copied_into_new;
+        } // end copy_slot
+    } // End of CHM
+
+    // --- Snapshot ------------------------------------------------------------
+    // The main class for iterating over the NBHM.  It "snapshots" a clean
+    // view of the K/V array.
+    private class SnapshotV implements Iterator, Enumeration {
+        final Object[] _sskvs;
+
+        public SnapshotV() {
+            while (true) { // Verify no table-copy-in-progress
+                Object[] topkvs = _kvs;
+                CHM topchm = chm(topkvs);
+                if (topchm._newkvs == null) { // No table-copy-in-progress
+                    // The "linearization point" for the iteration.  Every key in this
+                    // table will be visited, but keys added later might be skipped or
+                    // even be added to a following table (also not iterated over).
+                    _sskvs = topkvs;
+                    break;
+                }
+                // Table copy in-progress - so we cannot get a clean iteration.  We
+                // must help finish the table copy before we can start iterating.
+                topchm.help_copy_impl(NonBlockingHashMap.this, topkvs, true);
+            }
+            // Warm-up the iterator
+            next();
+        }
+
+        int length() {
+            return len(_sskvs);
+        }
+
+        Object key(int idx) {
+            return NonBlockingHashMap.key(_sskvs, idx);
+        }
+
+        private int _idx; // Varies from 0-keys.length
+        private Object _nextK, _prevK; // Last 2 keys found
+        private TypeV  _nextV, _prevV; // Last 2 values found
+
+        public boolean hasNext() {
+            return _nextV != null;
+        }
+
+        public TypeV next() {
+            // 'next' actually knows what the next value will be - it had to
+            // figure that out last go-around lest 'hasNext' report true and
+            // some other thread deleted the last value.  Instead, 'next'
+            // spends all its effort finding the key that comes after the
+            // 'next' key.
+            if (_idx != 0 && _nextV == null)
+                throw new NoSuchElementException();
+            _prevK = _nextK; // This will become the previous key
+            _prevV = _nextV; // This will become the previous value
+            _nextV = null; // We have no more next-key
+            // Attempt to set <_nextK,_nextV> to the next K,V pair.
+            // _nextV is the trigger: stop searching when it is != null
+            while (_idx < length()) { // Scan array
+                _nextK = key(_idx++); // Get a key that definitely is in the set (for the moment!)
+                if (_nextK != null && // Found something?
+                    _nextK != TOMBSTONE && (_nextV = get(_nextK)) != null)
+                    break; // Got it!  _nextK is a valid Key
+            } // Else keep scanning
+            return _prevV; // Return current value.
+        }
+
+        public void remove() {
+            if (_prevV == null)
+                throw new IllegalStateException();
+            putIfMatch(NonBlockingHashMap.this, _sskvs, _prevK, TOMBSTONE, _prevV);
+            _prevV = null;
+        }
+
+        public TypeV nextElement() {
+            return next();
+        }
+
+        public boolean hasMoreElements() {
+            return hasNext();
+        }
+    }
+
+    public Object[] raw_array() {
+        return new SnapshotV()._sskvs;
+    }
+
+    /**
+     * Returns an enumeration of the values in this table.
+     *
+     * @return an enumeration of the values in this table
+     * @see #values()
+     */
+    public Enumeration elements() {
+        return new SnapshotV();
+    }
+
+    // --- values --------------------------------------------------------------
+
+    /**
+     * Returns a {@link Collection} view of the values contained in this map.
+     * The collection is backed by the map, so changes to the map are reflected
+     * in the collection, and vice-versa.  The collection supports element
+     * removal, which removes the corresponding mapping from this map, via the
+     * Iterator.remove, Collection.remove,
+     * removeAll, retainAll, and clear operations.
+     * It does not support the add or addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator that
+     * will never throw {@link ConcurrentModificationException}, and guarantees
+     * to traverse elements as they existed upon construction of the iterator,
+     * and may (but is not guaranteed to) reflect any modifications subsequent
+     * to construction.
+     */
+    @Override
+    public Collection values() {
+        return new AbstractCollection() {
+            @Override
+            public void clear() {
+                NonBlockingHashMap.this.clear();
+            }
+
+            @Override
+            public int size() {
+                return NonBlockingHashMap.this.size();
+            }
+
+            @Override
+            public boolean contains(Object v) {
+                return NonBlockingHashMap.this.containsValue(v);
+            }
+
+            @Override
+            public Iterator iterator() {
+                return new SnapshotV();
+            }
+        };
+    }
+
+    // --- keySet --------------------------------------------------------------
+    private class SnapshotK implements Iterator, Enumeration {
+        final SnapshotV _ss;
+
+        public SnapshotK() {
+            _ss = new SnapshotV();
+        }
+
+        public void remove() {
+            _ss.remove();
+        }
+
+        @SuppressWarnings("unchecked")
+        public TypeK next() {
+            _ss.next();
+            return (TypeK) _ss._prevK;
+        }
+
+        public boolean hasNext() {
+            return _ss.hasNext();
+        }
+
+        public TypeK nextElement() {
+            return next();
+        }
+
+        public boolean hasMoreElements() {
+            return hasNext();
+        }
+    }
+
+    /**
+     * Returns an enumeration of the keys in this table.
+     *
+     * @return an enumeration of the keys in this table
+     * @see #keySet()
+     */
+    public Enumeration keys() {
+        return new SnapshotK();
+    }
+
+    /**
+     * Returns a {@link Set} view of the keys contained in this map.  The set
+     * is backed by the map, so changes to the map are reflected in the set,
+     * and vice-versa.  The set supports element removal, which removes the
+     * corresponding mapping from this map, via the Iterator.remove,
+     * Set.remove, removeAll, retainAll, and
+     * clear operations.  It does not support the add or
+     * addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator that
+     * will never throw {@link ConcurrentModificationException}, and guarantees
+     * to traverse elements as they existed upon construction of the iterator,
+     * and may (but is not guaranteed to) reflect any modifications subsequent
+     * to construction.
+     */
+    @Override
+    public Set keySet() {
+        return new AbstractSet() {
+            @Override
+            public void clear() {
+                NonBlockingHashMap.this.clear();
+            }
+
+            @Override
+            public int size() {
+                return NonBlockingHashMap.this.size();
+            }
+
+            @Override
+            public boolean contains(Object k) {
+                return NonBlockingHashMap.this.containsKey(k);
+            }
+
+            @Override
+            public boolean remove(Object k) {
+                return NonBlockingHashMap.this.remove(k) != null;
+            }
+
+            @Override
+            public Iterator iterator() {
+                return new SnapshotK();
+            }
+        };
+    }
+
+    // --- entrySet ------------------------------------------------------------
+    // Warning: Each call to 'next' in this iterator constructs a new NBHMEntry.
+    private class NBHMEntry extends AbstractEntry {
+        NBHMEntry(final TypeK k, final TypeV v) {
+            super(k, v);
+        }
+
+        public TypeV setValue(final TypeV val) {
+            if (val == null)
+                throw new NullPointerException();
+            _val = val;
+            return put(_key, val);
+        }
+    }
+
+    private class SnapshotE implements Iterator> {
+        final SnapshotV _ss;
+
+        public SnapshotE() {
+            _ss = new SnapshotV();
+        }
+
+        public void remove() {
+            _ss.remove();
+        }
+
+        @SuppressWarnings("unchecked")
+        public Entry next() {
+            _ss.next();
+            return new NBHMEntry((TypeK) _ss._prevK, _ss._prevV);
+        }
+
+        public boolean hasNext() {
+            return _ss.hasNext();
+        }
+    }
+
+    /**
+     * Returns a {@link Set} view of the mappings contained in this map.  The
+     * set is backed by the map, so changes to the map are reflected in the
+     * set, and vice-versa.  The set supports element removal, which removes
+     * the corresponding mapping from the map, via the
+     * Iterator.remove, Set.remove, removeAll,
+     * retainAll, and clear operations.  It does not support
+     * the add or addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator
+     * that will never throw {@link ConcurrentModificationException},
+     * and guarantees to traverse elements as they existed upon
+     * construction of the iterator, and may (but is not guaranteed to)
+     * reflect any modifications subsequent to construction.
+     * 
    
+     * 
    Warning: the iterator associated with this Set
+     * requires the creation of {@link Entry} objects with each
+     * iteration.  The {@link NonBlockingHashMap} does not normally create or
+     * using {@link Entry} objects so they will be created soley
+     * to support this iteration.  Iterating using keySet or values will be
+     * more efficient.
+     */
+    @Override
+    public Set> entrySet() {
+        return new AbstractSet>() {
+            @Override
+            public void clear() {
+                NonBlockingHashMap.this.clear();
+            }
+
+            @Override
+            public int size() {
+                return NonBlockingHashMap.this.size();
+            }
+
+            @Override
+            public boolean remove(final Object o) {
+                if (!(o instanceof Map.Entry))
+                    return false;
+                final Entry e = (Entry) o;
+                return NonBlockingHashMap.this.remove(e.getKey(), e.getValue());
+            }
+
+            @Override
+            public boolean contains(final Object o) {
+                if (!(o instanceof Map.Entry))
+                    return false;
+                final Entry e = (Entry) o;
+                TypeV v = get(e.getKey());
+                return v.equals(e.getValue());
+            }
+
+            @Override
+            public Iterator> iterator() {
+                return new SnapshotE();
+            }
+        };
+    }
+
+    // --- writeObject -------------------------------------------------------
+    // Write a NBHM to a stream
+    private void writeObject(java.io.ObjectOutputStream s) throws IOException {
+        s.defaultWriteObject(); // Nothing to write
+        for (Object K : keySet()) {
+            final Object V = get(K); // Do an official 'get'
+            s.writeObject(K); // Write the  pair
+            s.writeObject(V);
+        }
+        s.writeObject(null); // Sentinel to indicate end-of-data
+        s.writeObject(null);
+    }
+
+    // --- readObject --------------------------------------------------------
+    // Read a CHM from a stream
+    @SuppressWarnings("unchecked")
+    private void readObject(java.io.ObjectInputStream s) throws IOException, ClassNotFoundException {
+        s.defaultReadObject(); // Read nothing
+        initialize(MIN_SIZE);
+        for (;;) {
+            final TypeK K = (TypeK) s.readObject();
+            final TypeV V = (TypeV) s.readObject();
+            if (K == null)
+                break;
+            put(K, V); // Insert with an offical put
+        }
+    }
+
+} // End NonBlockingHashMap class
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMapLong.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMapLong.java
new file mode 100644
index 0000000..1ff0236
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/collection/NonBlockingHashMapLong.java
@@ -0,0 +1,1516 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.collection;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.lang.reflect.Field;
+import java.util.AbstractCollection;
+import java.util.AbstractMap;
+import java.util.AbstractSet;
+import java.util.Collection;
+import java.util.ConcurrentModificationException;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicLongFieldUpdater;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+
+import sun.misc.Unsafe;
+
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ * A lock-free alternate implementation of {@link java.util.concurrent.ConcurrentHashMap}
+ * with primitive long keys, better scaling properties and
+ * generally lower costs.  The use of {@code long} keys allows for faster
+ * compares and lower memory costs.  The Map provides identical correctness
+ * properties as ConcurrentHashMap.  All operations are non-blocking and
+ * multi-thread safe, including all update operations.  {@link
+ * NonBlockingHashMapLong} scales substatially better than {@link
+ * java.util.concurrent.ConcurrentHashMap} for high update rates, even with a large
+ * concurrency factor.  Scaling is linear up to 768 CPUs on a 768-CPU Azul
+ * box, even with 100% updates or 100% reads or any fraction in-between.
+ * Linear scaling up to all cpus has been observed on a 32-way Sun US2 box,
+ * 32-way Sun Niagra box, 8-way Intel box and a 4-way Power box.
+ * 
    
+ * 
    The main benefit of this class over using plain
+ * NonBlockingHashMap with {@link Long} keys is
+ * that it avoids the auto-boxing and unboxing costs.  Since auto-boxing is
+ * automatic, it is easy to accidentally cause auto-boxing and negate
+ * the space and speed benefits.
+ * 
    
+ * 
    This class obeys the same functional specification as {@link
+ * Hashtable}, and includes versions of methods corresponding to
+ * each method of Hashtable.  However, even though all operations are
+ * thread-safe, operations do not entail locking and there is
+ * not any support for locking the entire table in a way that
+ * prevents all access.  This class is fully interoperable with
+ * Hashtable in programs that rely on its thread safety but not on
+ * its synchronization details.
+ * 
    
+ * 
     Operations (including put) generally do not block, so may
+ * overlap with other update operations (including other puts and
+ * removes).  Retrievals reflect the results of the most recently
+ * completed update operations holding upon their onset.  For
+ * aggregate operations such as putAll, concurrent retrievals may
+ * reflect insertion or removal of only some entries.  Similarly, Iterators
+ * and Enumerations return elements reflecting the state of the hash table at
+ * some point at or since the creation of the iterator/enumeration.  They do
+ * not throw {@link ConcurrentModificationException}.  However,
+ * iterators are designed to be used by only one thread at a time.
+ * 
    
+ * 
     Very full tables, or tables with high reprobe rates may trigger an
+ * internal resize operation to move into a larger table.  Resizing is not
+ * terribly expensive, but it is not free either; during resize operations
+ * table throughput may drop somewhat.  All threads that visit the table
+ * during a resize will 'help' the resizing but will still be allowed to
+ * complete their operation before the resize is finished (i.e., a simple
+ * 'get' operation on a million-entry table undergoing resizing will not need
+ * to block until the entire million entries are copied).
+ * 
    
+ * 
    This class and its views and iterators implement all of the
+ * optional methods of the {@link Map} and {@link Iterator}
+ * interfaces.
+ * 
    
+ * 
     Like {@link Hashtable} but unlike {@link HashMap}, this class
+ * does not allow null to be used as a value.
+ *
+ * @param  the type of mapped values
+ * @author Cliff Click
+ * @since 1.5
+ *
+ * Forked from JCTools.
+ */
+public class NonBlockingHashMapLong extends AbstractMap implements ConcurrentMap,
+                                                                           Serializable {
+
+    private static final long serialVersionUID = 1234123412341234124L;
+
+    private static Unsafe     unsafe           = UnsafeUtil.getUnsafeAccessor().getUnsafe();
+
+    // Too many reprobes then force a table-resize
+    private static final int  REPROBE_LIMIT    = 10;
+
+    // --- Bits to allow Unsafe access to arrays
+    private static final int  _Obase           = unsafe.arrayBaseOffset(Object[].class);
+    private static final int  _Oscale          = unsafe.arrayIndexScale(Object[].class);
+
+    private static long rawIndex(final Object[] ary, final int idx) {
+        assert idx >= 0 && idx < ary.length;
+        return _Obase + ((long) idx * _Oscale);
+    }
+
+    private static final int _Lbase  = unsafe.arrayBaseOffset(long[].class);
+    private static final int _Lscale = unsafe.arrayIndexScale(long[].class);
+
+    private static long rawIndex(final long[] ary, final int idx) {
+        assert idx >= 0 && idx < ary.length;
+        return _Lbase + ((long) idx * _Lscale);
+    }
+
+    // --- Bits to allow Unsafe CAS'ing of the CHM field
+    private static final long _chm_offset;
+    private static final long _val_1_offset;
+
+    static { // 
+        Field f;
+        try {
+            f = NonBlockingHashMapLong.class.getDeclaredField("_chm");
+        } catch (NoSuchFieldException e) {
+            throw new RuntimeException(e);
+        }
+        _chm_offset = unsafe.objectFieldOffset(f);
+
+        try {
+            f = NonBlockingHashMapLong.class.getDeclaredField("_val_1");
+        } catch (NoSuchFieldException e) {
+            throw new RuntimeException(e);
+        }
+        _val_1_offset = unsafe.objectFieldOffset(f);
+    }
+
+    private boolean CAS(final long offset, final Object old, final Object nnn) {
+        return unsafe.compareAndSwapObject(this, offset, old, nnn);
+    }
+
+    // --- Adding a 'prime' bit onto Values via wrapping with a junk wrapper class
+    private static final class Prime {
+        final Object _V;
+
+        Prime(Object V) {
+            _V = V;
+        }
+
+        static Object unbox(Object V) {
+            return V instanceof Prime ? ((Prime) V)._V : V;
+        }
+    }
+
+    // --- The Hash Table --------------------
+    private transient CHM       _chm;
+    // This next field holds the value for Key 0 - the special key value which
+    // is the initial array value, and also means: no-key-inserted-yet.
+    private transient Object    _val_1;                             // Value for Key: NO_KEY
+
+    // Time since last resize
+    private transient long      _last_resize_milli;
+
+    // Optimize for space: use a 1/2-sized table and allow more re-probes
+    private final boolean       _opt_for_space;
+
+    // --- Minimum table size ----------------
+    // Pick size 16 K/V pairs, which turns into (16*2)*4+12 = 140 bytes on a
+    // standard 32-bit HotSpot, and (16*2)*8+12 = 268 bytes on 64-bit Azul.
+    private static final int    MIN_SIZE_LOG = 4;                   //
+    private static final int    MIN_SIZE     = (1 << MIN_SIZE_LOG); // Must be power of 2
+
+    // --- Sentinels -------------------------
+    // No-Match-Old - putIfMatch does updates only if it matches the old value,
+    // and NO_MATCH_OLD basically counts as a wildcard match.
+    private static final Object NO_MATCH_OLD = new Object();        // Sentinel
+    // Match-Any-not-null - putIfMatch does updates only if it find a real old
+    // value.
+    private static final Object MATCH_ANY    = new Object();        // Sentinel
+    // This K/V pair has been deleted (but the Key slot is forever claimed).
+    // The same Key can be reinserted with a new value later.
+    private static final Object TOMBSTONE    = new Object();
+    // Prime'd or box'd version of TOMBSTONE.  This K/V pair was deleted, then a
+    // table resize started.  The K/V pair has been marked so that no new
+    // updates can happen to the old table (and since the K/V pair was deleted
+    // nothing was copied to the new table).
+    private static final Prime  TOMBPRIME    = new Prime(TOMBSTONE);
+
+    // I exclude 1 long from the 2^64 possibilities, and test for it before
+    // entering the main array.  The NO_KEY value must be zero, the initial
+    // value set by Java before it hands me the array.
+    private static final long   NO_KEY       = 0L;
+
+    // --- dump ----------------------------------------------------------------
+
+    /**
+     * Verbose printout of table internals, useful for debugging.
+     */
+    public final void print() {
+        System.out.println("========="); // NOPMD
+        print_impl(-99, NO_KEY, _val_1);
+        _chm.print();
+        System.out.println("========="); // NOPMD
+    }
+
+    private static void print_impl(final int i, final long K, final Object V) {
+        String p = (V instanceof Prime) ? "prime_" : "";
+        Object V2 = Prime.unbox(V);
+        String VS = (V2 == TOMBSTONE) ? "tombstone" : V2.toString();
+        System.out.println("[" + i + "]=(" + K + "," + p + VS + ")"); // NOPMD
+    }
+
+    @SuppressWarnings("unused")
+    private void print2() {
+        System.out.println("========="); // NOPMD
+        print2_impl(-99, NO_KEY, _val_1);
+        _chm.print();
+        System.out.println("========="); // NOPMD
+    }
+
+    private static void print2_impl(final int i, final long K, final Object V) {
+        if (V != null && Prime.unbox(V) != TOMBSTONE)
+            print_impl(i, K, V);
+    }
+
+    // --- reprobe_limit -----------------------------------------------------
+    // Heuristic to decide if we have reprobed toooo many times.  Running over
+    // the reprobe limit on a 'get' call acts as a 'miss'; on a 'put' call it
+    // can trigger a table resize.  Several places must have exact agreement on
+    // what the reprobe_limit is, so we share it here.
+    private static int reprobe_limit(int len) {
+        return REPROBE_LIMIT + (len >> 8);
+    }
+
+    // --- NonBlockingHashMapLong ----------------------------------------------
+    // Constructors
+
+    /**
+     * Create a new NonBlockingHashMapLong with default minimum size (currently set
+     * to 8 K/V pairs or roughly 84 bytes on a standard 32-bit JVM).
+     */
+    public NonBlockingHashMapLong() {
+        this(MIN_SIZE, true);
+    }
+
+    /**
+     * Create a new NonBlockingHashMapLong with initial room for the given
+     * number of elements, thus avoiding internal resizing operations to reach
+     * an appropriate size.  Large numbers here when used with a small count of
+     * elements will sacrifice space for a small amount of time gained.  The
+     * initial size will be rounded up internally to the next larger power of 2.
+     */
+    public NonBlockingHashMapLong(final int initial_sz) {
+        this(initial_sz, true);
+    }
+
+    /**
+     * Create a new NonBlockingHashMapLong, setting the space-for-speed
+     * tradeoff.  {@code true} optimizes for space and is the default.  {@code
+     * false} optimizes for speed and doubles space costs for roughly a 10%
+     * speed improvement.
+     */
+    public NonBlockingHashMapLong(final boolean opt_for_space) {
+        this(1, opt_for_space);
+    }
+
+    /**
+     * Create a new NonBlockingHashMapLong, setting both the initial size and
+     * the space-for-speed tradeoff.  {@code true} optimizes for space and is
+     * the default.  {@code false} optimizes for speed and doubles space costs
+     * for roughly a 10% speed improvement.
+     */
+    public NonBlockingHashMapLong(final int initial_sz, final boolean opt_for_space) {
+        _opt_for_space = opt_for_space;
+        initialize(initial_sz);
+    }
+
+    @SuppressWarnings("StatementWithEmptyBody")
+    private void initialize(final int initial_sz) {
+        if (initial_sz < 0)
+            throw new IllegalArgumentException();
+        int i; // Convert to next largest power-of-2
+        for (i = MIN_SIZE_LOG; (1 << i) < initial_sz; i++) { /* empty */
+        }
+        _chm = new CHM(this, new ConcurrentAutoTable(), i);
+        _val_1 = TOMBSTONE; // Always as-if deleted
+        _last_resize_milli = System.currentTimeMillis();
+    }
+
+    // --- wrappers ------------------------------------------------------------
+
+    /**
+     * Returns the number of key-value mappings in this map.
+     *
+     * @return the number of key-value mappings in this map
+     */
+    public int size() {
+        return (_val_1 == TOMBSTONE ? 0 : 1) + _chm.size();
+    }
+
+    /**
+     * Tests if the key in the table.
+     *
+     * @return true if the key is in the table
+     */
+    public boolean containsKey(long key) {
+        return get(key) != null;
+    }
+
+    /**
+     * Legacy method testing if some key maps into the specified value in this
+     * table.  This method is identical in functionality to {@link
+     * #containsValue}, and exists solely to ensure full compatibility with
+     * class {@link Hashtable}, which supported this method prior to
+     * introduction of the Java Collections framework.
+     *
+     * @param val a value to search for
+     * @return true if this map maps one or more keys to the specified value
+     * @throws NullPointerException if the specified value is null
+     */
+    public boolean contains(Object val) {
+        return containsValue(val);
+    }
+
+    /**
+     * Maps the specified key to the specified value in the table.  The value
+     * cannot be null.  
     The value can be retrieved by calling {@link #get}
+     * with a key that is equal to the original key.
+     *
+     * @param key key with which the specified value is to be associated
+     * @param val value to be associated with the specified key
+     * @return the previous value associated with key, or
+     * null if there was no mapping for key
+     * @throws NullPointerException if the specified value is null
+     */
+    public TypeV put(long key, TypeV val) {
+        return putIfMatch(key, val, NO_MATCH_OLD);
+    }
+
+    /**
+     * Atomically, do a {@link #put} if-and-only-if the key is not mapped.
+     * Useful to ensure that only a single mapping for the key exists, even if
+     * many threads are trying to create the mapping in parallel.
+     *
+     * @return the previous value associated with the specified key,
+     * or null if there was no mapping for the key
+     * @throws NullPointerException if the specified is value is null
+     */
+    public TypeV putIfAbsent(long key, TypeV val) {
+        return putIfMatch(key, val, TOMBSTONE);
+    }
+
+    /**
+     * Removes the key (and its corresponding value) from this map.
+     * This method does nothing if the key is not in the map.
+     *
+     * @return the previous value associated with key, or
+     * null if there was no mapping for key
+     */
+    public TypeV remove(long key) {
+        return putIfMatch(key, TOMBSTONE, NO_MATCH_OLD);
+    }
+
+    /**
+     * Atomically do a {@link #remove(long)} if-and-only-if the key is mapped
+     * to a value which is equals to the given value.
+     *
+     * @throws NullPointerException if the specified value is null
+     */
+    public boolean remove(long key, Object val) {
+        return putIfMatch(key, TOMBSTONE, val) == val;
+    }
+
+    /**
+     * Atomically do a put(key,val) if-and-only-if the key is
+     * mapped to some value already.
+     *
+     * @throws NullPointerException if the specified value is null
+     */
+    public TypeV replace(long key, TypeV val) {
+        return putIfMatch(key, val, MATCH_ANY);
+    }
+
+    /**
+     * Atomically do a put(key,newValue) if-and-only-if the key is
+     * mapped a value which is equals to oldValue.
+     *
+     * @throws NullPointerException if the specified value is null
+     */
+    public boolean replace(long key, TypeV oldValue, TypeV newValue) {
+        return putIfMatch(key, newValue, oldValue) == oldValue;
+    }
+
+    @SuppressWarnings("unchecked")
+    private TypeV putIfMatch(long key, Object newVal, Object oldVal) {
+        if (oldVal == null || newVal == null)
+            throw new NullPointerException();
+        if (key == NO_KEY) {
+            Object curVal = _val_1;
+            if (oldVal == NO_MATCH_OLD || // Do we care about expected-Value at all?
+                curVal == oldVal || // No instant match already?
+                (oldVal == MATCH_ANY && curVal != TOMBSTONE) || oldVal.equals(curVal)) { // Expensive equals check
+                if (!CAS(_val_1_offset, curVal, newVal)) // One shot CAS update attempt
+                    curVal = _val_1; // Failed; get failing witness
+            }
+            return curVal == TOMBSTONE ? null : (TypeV) curVal; // Return the last value present
+        }
+        final Object res = _chm.putIfMatch(key, newVal, oldVal);
+        assert !(res instanceof Prime);
+        assert res != null;
+        return res == TOMBSTONE ? null : (TypeV) res;
+    }
+
+    /**
+     * Removes all of the mappings from this map.
+     */
+    @SuppressWarnings("StatementWithEmptyBody")
+    public void clear() { // Smack a new empty table down
+        CHM newchm = new CHM(this, new ConcurrentAutoTable(), MIN_SIZE_LOG);
+        while (!CAS(_chm_offset, _chm, newchm)) { // NOPMD
+            /* Spin until the clear works */
+        }
+        CAS(_val_1_offset, _val_1, TOMBSTONE);
+    }
+
+    /**
+     * Returns true if this Map maps one or more keys to the specified
+     * value.  Note: This method requires a full internal traversal of the
+     * hash table and is much slower than {@link #containsKey}.
+     *
+     * @param val value whose presence in this map is to be tested
+     * @return true if this Map maps one or more keys to the specified value
+     * @throws NullPointerException if the specified value is null
+     */
+    public boolean containsValue(Object val) {
+        if (val == null)
+            return false;
+        if (val == _val_1)
+            return true; // Key 0
+        for (TypeV V : values())
+            if (V == val || V.equals(val))
+                return true;
+        return false;
+    }
+
+    // --- get -----------------------------------------------------------------
+
+    /**
+     * Returns the value to which the specified key is mapped, or {@code null}
+     * if this map contains no mapping for the key.
+     * 
    More formally, if this map contains a mapping from a key {@code k} to
+     * a value {@code v} such that {@code key==k}, then this method
+     * returns {@code v}; otherwise it returns {@code null}.  (There can be at
+     * most one such mapping.)
+     *
+     * @throws NullPointerException if the specified key is null
+     */
+    // Never returns a Prime nor a Tombstone.
+    @SuppressWarnings("unchecked")
+    public final TypeV get(long key) {
+        if (key == NO_KEY) {
+            final Object V = _val_1;
+            return V == TOMBSTONE ? null : (TypeV) V;
+        }
+        final Object V = _chm.get_impl(key);
+        assert !(V instanceof Prime); // Never return a Prime
+        assert V != TOMBSTONE;
+        return (TypeV) V;
+    }
+
+    /**
+     * Auto-boxing version of {@link #get(long)}.
+     */
+    public TypeV get(Object key) {
+        return (key instanceof Long) ? get(((Long) key).longValue()) : null;
+    }
+
+    /**
+     * Auto-boxing version of {@link #remove(long)}.
+     */
+    public TypeV remove(Object key) {
+        return (key instanceof Long) ? remove(((Long) key).longValue()) : null;
+    }
+
+    /**
+     * Auto-boxing version of {@link #remove(long, Object)}.
+     */
+    public boolean remove(Object key, Object Val) {
+        return (key instanceof Long) && remove(((Long) key).longValue(), Val);
+    }
+
+    /**
+     * Auto-boxing version of {@link #containsKey(long)}.
+     */
+    public boolean containsKey(Object key) {
+        return (key instanceof Long) && containsKey(((Long) key).longValue());
+    }
+
+    /**
+     * Auto-boxing version of {@link #putIfAbsent}.
+     */
+    public TypeV putIfAbsent(Long key, TypeV val) {
+        return putIfAbsent(key.longValue(), val);
+    }
+
+    /**
+     * Auto-boxing version of {@link #replace}.
+     */
+    public TypeV replace(Long key, TypeV Val) {
+        return replace(key.longValue(), Val);
+    }
+
+    /**
+     * Auto-boxing version of {@link #put}.
+     */
+    public TypeV put(Long key, TypeV val) {
+        return put(key.longValue(), val);
+    }
+
+    /**
+     * Auto-boxing version of {@link #replace}.
+     */
+    public boolean replace(Long key, TypeV oldValue, TypeV newValue) {
+        return replace(key.longValue(), oldValue, newValue);
+    }
+
+    // --- help_copy -----------------------------------------------------------
+    // Help along an existing resize operation.  This is just a fast cut-out
+    // wrapper, to encourage inlining for the fast no-copy-in-progress case.  We
+    // always help the top-most table copy, even if there are nested table
+    // copies in progress.
+    private void help_copy() {
+        // Read the top-level CHM only once.  We'll try to help this copy along,
+        // even if it gets promoted out from under us (i.e., the copy completes
+        // and another KVS becomes the top-level copy).
+        CHM topchm = _chm;
+        if (topchm._newchm == null)
+            return; // No copy in-progress
+        topchm.help_copy_impl(false);
+    }
+
+    // --- CHM -----------------------------------------------------------------
+    // The control structure for the NonBlockingHashMapLong
+    private static final class CHM implements Serializable {
+
+        private static final long         serialVersionUID = 4958597092775229086L;
+
+        // Back-pointer to top-level structure
+        final NonBlockingHashMapLong      _nbhml;
+
+        // Size in active K,V pairs
+        private final ConcurrentAutoTable _size;
+
+        public int size() {
+            return (int) _size.get();
+        }
+
+        // ---
+        // These next 2 fields are used in the resizing heuristics, to judge when
+        // it is time to resize or copy the table.  Slots is a count of used-up
+        // key slots, and when it nears a large fraction of the table we probably
+        // end up reprobing too much.  Last-resize-milli is the time since the
+        // last resize; if we are running back-to-back resizes without growing
+        // (because there are only a few live keys but many slots full of dead
+        // keys) then we need a larger table to cut down on the churn.
+
+        // Count of used slots, to tell when table is full of dead unusable slots
+        private final ConcurrentAutoTable _slots;
+
+        @SuppressWarnings("unused")
+        public int slots() {
+            return (int) _slots.get();
+        }
+
+        // ---
+        // New mappings, used during resizing.
+        // The 'next' CHM - created during a resize operation.  This represents
+        // the new table being copied from the old one.  It's the volatile
+        // variable that is read as we cross from one table to the next, to get
+        // the required memory orderings.  It monotonically transits from null to
+        // set (once).
+        volatile CHM                                               _newchm;
+        private static final AtomicReferenceFieldUpdater _newchmUpdater = AtomicReferenceFieldUpdater
+                                                                                      .newUpdater(CHM.class, CHM.class,
+                                                                                          "_newchm");
+
+        // Set the _newchm field if we can.  AtomicUpdaters do not fail spuriously.
+        boolean CAS_newchm(CHM newchm) {
+            return _newchmUpdater.compareAndSet(this, null, newchm);
+        }
+
+        // Sometimes many threads race to create a new very large table.  Only 1
+        // wins the race, but the losers all allocate a junk large table with
+        // hefty allocation costs.  Attempt to control the overkill here by
+        // throttling attempts to create a new table.  I cannot really block here
+        // (lest I lose the non-blocking property) but late-arriving threads can
+        // give the initial resizing thread a little time to allocate the initial
+        // new table.  The Right Long Term Fix here is to use array-lets and
+        // incrementally create the new very large array.  In C I'd make the array
+        // with malloc (which would mmap under the hood) which would only eat
+        // virtual-address and not real memory - and after Somebody wins then we
+        // could in parallel initialize the array.  Java does not allow
+        // un-initialized array creation (especially of ref arrays!).
+        volatile long                                    _resizers;                       // count of threads attempting an initial resize
+        private static final AtomicLongFieldUpdater _resizerUpdater = AtomicLongFieldUpdater.newUpdater(CHM.class,
+                                                                             "_resizers");
+
+        // --- key,val -------------------------------------------------------------
+        // Access K,V for a given idx
+        @SuppressWarnings("SameParameterValue")
+        private boolean CAS_key(int idx, long old, long key) {
+            return unsafe.compareAndSwapLong(_keys, rawIndex(_keys, idx), old, key);
+        }
+
+        private boolean CAS_val(int idx, Object old, Object val) {
+            return unsafe.compareAndSwapObject(_vals, rawIndex(_vals, idx), old, val);
+        }
+
+        final long[]   _keys;
+        final Object[] _vals;
+
+        // Simple constructor
+        CHM(final NonBlockingHashMapLong nbhml, ConcurrentAutoTable size, final int logsize) {
+            _nbhml = nbhml;
+            _size = size;
+            _slots = new ConcurrentAutoTable();
+            _keys = new long[1 << logsize];
+            _vals = new Object[1 << logsize];
+        }
+
+        // --- print innards
+        private void print() {
+            for (int i = 0; i < _keys.length; i++) {
+                long K = _keys[i];
+                if (K != NO_KEY)
+                    print_impl(i, K, _vals[i]);
+            }
+            CHM newchm = _newchm; // New table, if any
+            if (newchm != null) {
+                System.out.println("----"); // NOPMD
+                newchm.print();
+            }
+        }
+
+        // --- print only the live objects
+        @SuppressWarnings("unused")
+        private void print2() {
+            for (int i = 0; i < _keys.length; i++) {
+                long K = _keys[i];
+                if (K != NO_KEY) // key is sane
+                    print2_impl(i, K, _vals[i]);
+            }
+            CHM newchm = _newchm; // New table, if any
+            if (newchm != null) {
+                System.out.println("----"); // NOPMD
+                newchm.print2();
+            }
+        }
+
+        // --- get_impl ----------------------------------------------------------
+        // Never returns a Prime nor a Tombstone.
+        private Object get_impl(final long key) {
+            final int len = _keys.length;
+            int idx = (int) (key & (len - 1)); // First key hash
+
+            // Main spin/reprobe loop, looking for a Key hit
+            int reprobe_cnt = 0;
+            while (true) {
+                final long K = _keys[idx]; // Get key   before volatile read, could be NO_KEY
+                final Object V = _vals[idx]; // Get value before volatile read, could be null or Tombstone or Prime
+                if (K == NO_KEY)
+                    return null; // A clear miss
+
+                // Key-compare
+                if (key == K) {
+                    // Key hit!  Check for no table-copy-in-progress
+                    if (!(V instanceof Prime)) { // No copy?
+                        if (V == TOMBSTONE)
+                            return null;
+                        // We need a volatile-read between reading a newly inserted Value
+                        // and returning the Value (so the user might end up reading the
+                        // stale Value contents).
+                        @SuppressWarnings("unused")
+                        final CHM newchm = _newchm; // VOLATILE READ before returning V
+                        return V;
+                    }
+                    // Key hit - but slot is (possibly partially) copied to the new table.
+                    // Finish the copy & retry in the new table.
+                    return copy_slot_and_check(idx, key).get_impl(key); // Retry in the new table
+                }
+                // get and put must have the same key lookup logic!  But only 'put'
+                // needs to force a table-resize for a too-long key-reprobe sequence.
+                // Check for too-many-reprobes on get.
+                if (++reprobe_cnt >= reprobe_limit(len)) // too many probes
+                    return _newchm == null // Table copy in progress?
+                    ? null // Nope!  A clear miss
+                        : copy_slot_and_check(idx, key).get_impl(key); // Retry in the new table
+
+                idx = (idx + 1) & (len - 1); // Reprobe by 1!  (could now prefetch)
+            }
+        }
+
+        // --- putIfMatch ---------------------------------------------------------
+        // Put, Remove, PutIfAbsent, etc.  Return the old value.  If the returned
+        // value is equal to expVal (or expVal is NO_MATCH_OLD) then the put can
+        // be assumed to work (although might have been immediately overwritten).
+        // Only the path through copy_slot passes in an expected value of null,
+        // and putIfMatch only returns a null if passed in an expected null.
+        private Object putIfMatch(final long key, final Object putval, final Object expVal) {
+            assert putval != null;
+            assert !(putval instanceof Prime);
+            assert !(expVal instanceof Prime);
+            final int len = _keys.length;
+            int idx = (int) (key & (len - 1)); // The first key
+
+            // ---
+            // Key-Claim stanza: spin till we can claim a Key (or force a resizing).
+            int reprobe_cnt = 0;
+            long K;
+            Object V;
+            while (true) { // Spin till we get a Key slot
+                V = _vals[idx]; // Get old value
+                K = _keys[idx]; // Get current key
+                if (K == NO_KEY) { // Slot is free?
+                    // Found an empty Key slot - which means this Key has never been in
+                    // this table.  No need to put a Tombstone - the Key is not here!
+                    if (putval == TOMBSTONE)
+                        return putval; // Not-now & never-been in this table
+                    // Claim the zero key-slot
+                    if (CAS_key(idx, NO_KEY, key)) { // Claim slot for Key
+                        _slots.add(1); // Raise key-slots-used count
+                        break; // Got it!
+                    }
+                    // CAS to claim the key-slot failed.
+                    //
+                    // This re-read of the Key points out an annoying short-coming of Java
+                    // CAS.  Most hardware CAS's report back the existing value - so that
+                    // if you fail you have a *witness* - the value which caused the CAS
+                    // to fail.  The Java API turns this into a boolean destroying the
+                    // witness.  Re-reading does not recover the witness because another
+                    // thread can write over the memory after the CAS.  Hence we can be in
+                    // the unfortunate situation of having a CAS fail *for cause* but
+                    // having that cause removed by a later store.  This turns a
+                    // non-spurious-failure CAS (such as Azul has) into one that can
+                    // apparently spuriously fail - and we avoid apparent spurious failure
+                    // by not allowing Keys to ever change.
+                    K = _keys[idx]; // CAS failed, get updated value
+                    assert K != NO_KEY; // If keys[idx] is NO_KEY, CAS shoulda worked
+                }
+                // Key slot was not null, there exists a Key here
+                if (K == key)
+                    break; // Got it!
+
+                // get and put must have the same key lookup logic!  Lest 'get' give
+                // up looking too soon.
+                //topmap._reprobes.add(1);
+                if (++reprobe_cnt >= reprobe_limit(len)) {
+                    // We simply must have a new table to do a 'put'.  At this point a
+                    // 'get' will also go to the new table (if any).  We do not need
+                    // to claim a key slot (indeed, we cannot find a free one to claim!).
+                    final CHM newchm = resize();
+                    if (expVal != null)
+                        _nbhml.help_copy(); // help along an existing copy
+                    return newchm.putIfMatch(key, putval, expVal);
+                }
+
+                idx = (idx + 1) & (len - 1); // Reprobe!
+            } // End of spinning till we get a Key slot
+
+            // ---
+            // Found the proper Key slot, now update the matching Value slot.  We
+            // never put a null, so Value slots monotonically move from null to
+            // not-null (deleted Values use Tombstone).  Thus if 'V' is null we
+            // fail this fast cutout and fall into the check for table-full.
+            if (putval == V)
+                return V; // Fast cutout for no-change
+
+            // See if we want to move to a new table (to avoid high average re-probe
+            // counts).  We only check on the initial set of a Value from null to
+            // not-null (i.e., once per key-insert).
+            if ((V == null && tableFull(reprobe_cnt, len)) ||
+            // Or we found a Prime: resize is already in progress.  The resize
+            // call below will do a CAS on _newchm forcing the read.
+                V instanceof Prime) {
+                resize(); // Force the new table copy to start
+                return copy_slot_and_check(idx, expVal).putIfMatch(key, putval, expVal);
+            }
+            //assert !(V instanceof Prime); // always true, so IDE warnings if uncommented
+
+            // ---
+            // We are finally prepared to update the existing table
+
+            // Must match old, and we do not?  Then bail out now.  Note that either V
+            // or expVal might be TOMBSTONE.  Also V can be null, if we've never
+            // inserted a value before.  expVal can be null if we are called from
+            // copy_slot.
+
+            if (expVal != NO_MATCH_OLD && // Do we care about expected-Value at all?
+                V != expVal && // No instant match already?
+                (expVal != MATCH_ANY || V == TOMBSTONE || V == null) && !(V == null && expVal == TOMBSTONE) && // Match on null/TOMBSTONE combo
+                (expVal == null || !expVal.equals(V))) // Expensive equals check at the last
+                return V; // Do not update!
+
+            // Actually change the Value in the Key,Value pair
+            if (CAS_val(idx, V, putval)) {
+                // CAS succeeded - we did the update!
+                // Both normal put's and table-copy calls putIfMatch, but table-copy
+                // does not (effectively) increase the number of live k/v pairs.
+                if (expVal != null) {
+                    // Adjust sizes - a striped counter
+                    if ((V == null || V == TOMBSTONE) && putval != TOMBSTONE)
+                        _size.add(1);
+                    if (!(V == null || V == TOMBSTONE) && putval == TOMBSTONE)
+                        _size.add(-1);
+                }
+            } else { // Else CAS failed
+                V = _vals[idx]; // Get new value
+                // If a Prime'd value got installed, we need to re-run the put on the
+                // new table.  Otherwise we lost the CAS to another racing put.
+                // Simply retry from the start.
+                if (V instanceof Prime)
+                    return copy_slot_and_check(idx, expVal).putIfMatch(key, putval, expVal);
+            }
+            // Win or lose the CAS, we are done.  If we won then we know the update
+            // happened as expected.  If we lost, it means "we won but another thread
+            // immediately stomped our update with no chance of a reader reading".
+            return (V == null && expVal != null) ? TOMBSTONE : V;
+        }
+
+        // --- tableFull ---------------------------------------------------------
+        // Heuristic to decide if this table is too full, and we should start a
+        // new table.  Note that if a 'get' call has reprobed too many times and
+        // decided the table must be full, then always the estimate_sum must be
+        // high and we must report the table is full.  If we do not, then we might
+        // end up deciding that the table is not full and inserting into the
+        // current table, while a 'get' has decided the same key cannot be in this
+        // table because of too many reprobes.  The invariant is:
+        //   slots.estimate_sum >= max_reprobe_cnt >= reprobe_limit(len)
+        private boolean tableFull(int reprobe_cnt, int len) {
+            return
+            // Do the cheap check first: we allow some number of reprobes always
+            reprobe_cnt >= REPROBE_LIMIT &&
+            // More expensive check: see if the table is > 1/2 full.
+                    _slots.estimate_get() >= reprobe_limit(len) * 2;
+        }
+
+        // --- resize ------------------------------------------------------------
+        // Resizing after too many probes.  "How Big???" heuristics are here.
+        // Callers will (not this routine) will 'help_copy' any in-progress copy.
+        // Since this routine has a fast cutout for copy-already-started, callers
+        // MUST 'help_copy' lest we have a path which forever runs through
+        // 'resize' only to discover a copy-in-progress which never progresses.
+        @SuppressWarnings("StatementWithEmptyBody")
+        private CHM resize() {
+            // Check for resize already in progress, probably triggered by another thread
+            CHM newchm = _newchm; // VOLATILE READ
+            if (newchm != null) // See if resize is already in progress
+                return newchm; // Use the new table already
+
+            // No copy in-progress, so start one.  First up: compute new table size.
+            int oldlen = _keys.length; // Old count of K,V pairs allowed
+            int sz = size(); // Get current table count of active K,V pairs
+            int newsz = sz; // First size estimate
+
+            // Heuristic to determine new size.  We expect plenty of dead-slots-with-keys
+            // and we need some decent padding to avoid endless reprobing.
+            if (_nbhml._opt_for_space) {
+                // This heuristic leads to a much denser table with a higher reprobe rate
+                if (sz >= (oldlen >> 1)) // If we are >50% full of keys then...
+                    newsz = oldlen << 1; // Double size
+            } else {
+                if (sz >= (oldlen >> 2)) { // If we are >25% full of keys then...
+                    newsz = oldlen << 1; // Double size
+                    if (sz >= (oldlen >> 1)) // If we are >50% full of keys then...
+                        newsz = oldlen << 2; // Double double size
+                }
+            }
+
+            // Last (re)size operation was very recent?  Then double again; slows
+            // down resize operations for tables subject to a high key churn rate.
+            long tm = System.currentTimeMillis();
+            if (newsz <= oldlen && // New table would shrink or hold steady?
+                tm <= _nbhml._last_resize_milli + 10000 // Recent resize (less than 1 sec ago)
+            //(q=_slots.estimate_sum()) >= (sz<<1) ) // 1/2 of keys are dead?
+            )
+                newsz = oldlen << 1; // Double the existing size
+
+            // Do not shrink, ever
+            if (newsz < oldlen)
+                newsz = oldlen;
+            //System.out.println("old="+oldlen+" new="+newsz+" size()="+sz+" est_slots()="+q+" millis="+(tm-_nbhml._last_resize_milli));
+
+            // Convert to power-of-2
+            int log2;
+            for (log2 = MIN_SIZE_LOG; (1 << log2) < newsz; log2++)
+                ; // Compute log2 of size
+
+            // Now limit the number of threads actually allocating memory to a
+            // handful - lest we have 750 threads all trying to allocate a giant
+            // resized array.
+            long r = _resizers;
+            while (!_resizerUpdater.compareAndSet(this, r, r + 1))
+                r = _resizers;
+            // Size calculation: 2 words (K+V) per table entry, plus a handful.  We
+            // guess at 32-bit pointers; 64-bit pointers screws up the size calc by
+            // 2x but does not screw up the heuristic very much.
+            int megs = ((((1 << log2) << 1) + 4) << 3/*word to bytes*/) >> 20/*megs*/;
+            if (r >= 2 && megs > 0) { // Already 2 guys trying; wait and see
+                newchm = _newchm; // Between dorking around, another thread did it
+                if (newchm != null) // See if resize is already in progress
+                    return newchm; // Use the new table already
+                // We could use a wait with timeout, so we'll wakeup as soon as the new table
+                // is ready, or after the timeout in any case.
+                //synchronized( this ) { wait(8*megs); }         // Timeout - we always wakeup
+                // For now, sleep a tad and see if the 2 guys already trying to make
+                // the table actually get around to making it happen.
+                try {
+                    Thread.sleep(8L * megs);
+                } catch (Exception e) { /*empty*/
+                }
+            }
+            // Last check, since the 'new' below is expensive and there is a chance
+            // that another thread slipped in a new thread while we ran the heuristic.
+            newchm = _newchm;
+            if (newchm != null) // See if resize is already in progress
+                return newchm; // Use the new table already
+
+            // New CHM - actually allocate the big arrays
+            newchm = new CHM(_nbhml, _size, log2);
+
+            // Another check after the slow allocation
+            if (_newchm != null) // See if resize is already in progress
+                return _newchm; // Use the new table already
+
+            // The new table must be CAS'd in so only 1 winner amongst duplicate
+            // racing resizing threads.  Extra CHM's will be GC'd.
+            if (!CAS_newchm(newchm)) { // NOW a resize-is-in-progress!
+                // CAS failed?
+                newchm = _newchm; // Reread new table
+            }
+            return newchm;
+        }
+
+        // The next part of the table to copy.  It monotonically transits from zero
+        // to _keys.length.  Visitors to the table can claim 'work chunks' by
+        // CAS'ing this field up, then copying the indicated indices from the old
+        // table to the new table.  Workers are not required to finish any chunk;
+        // the counter simply wraps and work is copied duplicately until somebody
+        // somewhere completes the count.
+        volatile long                                    _copyIdx         = 0;
+        static private final AtomicLongFieldUpdater _copyIdxUpdater  = AtomicLongFieldUpdater.newUpdater(
+                                                                              CHM.class, "_copyIdx");
+
+        // Work-done reporting.  Used to efficiently signal when we can move to
+        // the new table.  From 0 to len(oldkvs) refers to copying from the old
+        // table to the new.
+        volatile long                                    _copyDone        = 0;
+        static private final AtomicLongFieldUpdater _copyDoneUpdater = AtomicLongFieldUpdater.newUpdater(
+                                                                              CHM.class, "_copyDone");
+
+        // --- help_copy_impl ----------------------------------------------------
+        // Help along an existing resize operation.  We hope its the top-level
+        // copy (it was when we started) but this CHM might have been promoted out
+        // of the top position.
+        private void help_copy_impl(final boolean copy_all) {
+            final CHM newchm = _newchm;
+            assert newchm != null; // Already checked by caller
+            int oldlen = _keys.length; // Total amount to copy
+            final int MIN_COPY_WORK = Math.min(oldlen, 1024); // Limit per-thread work
+
+            // ---
+            int panic_start = -1;
+            int copyidx = -9999; // Fool javac to think it's initialized
+            while (_copyDone < oldlen) { // Still needing to copy?
+                // Carve out a chunk of work.  The counter wraps around so every
+                // thread eventually tries to copy every slot repeatedly.
+
+                // We "panic" if we have tried TWICE to copy every slot - and it still
+                // has not happened.  i.e., twice some thread somewhere claimed they
+                // would copy 'slot X' (by bumping _copyIdx) but they never claimed to
+                // have finished (by bumping _copyDone).  Our choices become limited:
+                // we can wait for the work-claimers to finish (and become a blocking
+                // algorithm) or do the copy work ourselves.  Tiny tables with huge
+                // thread counts trying to copy the table often 'panic'.
+                if (panic_start == -1) { // No panic?
+                    copyidx = (int) _copyIdx;
+                    while (copyidx < (oldlen << 1) && // 'panic' check
+                           !_copyIdxUpdater.compareAndSet(this, copyidx, copyidx + MIN_COPY_WORK))
+                        copyidx = (int) _copyIdx; // Re-read
+                    if (!(copyidx < (oldlen << 1))) // Panic!
+                        panic_start = copyidx; // Record where we started to panic-copy
+                }
+
+                // We now know what to copy.  Try to copy.
+                int workdone = 0;
+                for (int i = 0; i < MIN_COPY_WORK; i++)
+                    if (copy_slot((copyidx + i) & (oldlen - 1))) // Made an oldtable slot go dead?
+                        workdone++; // Yes!
+                if (workdone > 0) // Report work-done occasionally
+                    copy_check_and_promote(workdone);// See if we can promote
+                //for( int i=0; i 0) {
+                while (!_copyDoneUpdater.compareAndSet(this, copyDone, nowDone)) {
+                    copyDone = _copyDone; // Reload, retry
+                    nowDone = copyDone + workdone;
+                    assert nowDone <= oldlen;
+                }
+                //if( (10*copyDone/oldlen) != (10*nowDone/oldlen) )
+                //  System.out.print(" "+nowDone*100/oldlen+"%"+"_"+(_copyIdx*100/oldlen)+"%");
+            }
+
+            // Check for copy being ALL done, and promote.  Note that we might have
+            // nested in-progress copies and manage to finish a nested copy before
+            // finishing the top-level copy.  We only promote top-level copies.
+            if (nowDone == oldlen && // Ready to promote this table?
+                _nbhml._chm == this && // Looking at the top-level table?
+                // Attempt to promote
+                _nbhml.CAS(_chm_offset, this, _newchm)) {
+                _nbhml._last_resize_milli = System.currentTimeMillis(); // Record resize time for next check
+                //long nano = System.nanoTime();
+                //System.out.println(" "+nano+" Promote table "+oldlen+" to "+_newchm._keys.length);
+                //System.out.print("_"+oldlen+"]");
+            }
+        }
+
+        // --- copy_slot ---------------------------------------------------------
+        // Copy one K/V pair from oldkvs[i] to newkvs.  Returns true if we can
+        // confirm that the new table guaranteed has a value for this old-table
+        // slot.  We need an accurate confirmed-copy count so that we know when we
+        // can promote (if we promote the new table too soon, other threads may
+        // 'miss' on values not-yet-copied from the old table).  We don't allow
+        // any direct updates on the new table, unless they first happened to the
+        // old table - so that any transition in the new table from null to
+        // not-null must have been from a copy_slot (or other old-table overwrite)
+        // and not from a thread directly writing in the new table.  Thus we can
+        // count null-to-not-null transitions in the new table.
+        private boolean copy_slot(int idx) {
+            // Blindly set the key slot from NO_KEY to some key which hashes here,
+            // to eagerly stop fresh put's from inserting new values in the old
+            // table when the old table is mid-resize.  We don't need to act on the
+            // results here, because our correctness stems from box'ing the Value
+            // field.  Slamming the Key field is a minor speed optimization.
+            long key;
+            while ((key = _keys[idx]) == NO_KEY)
+                CAS_key(idx, NO_KEY, (idx + _keys.length)/*a non-zero key which hashes here*/);
+
+            // ---
+            // Prevent new values from appearing in the old table.
+            // Box what we see in the old table, to prevent further updates.
+            Object oldval = _vals[idx]; // Read OLD table
+            while (!(oldval instanceof Prime)) {
+                final Prime box = (oldval == null || oldval == TOMBSTONE) ? TOMBPRIME : new Prime(oldval);
+                if (CAS_val(idx, oldval, box)) { // CAS down a box'd version of oldval
+                    // If we made the Value slot hold a TOMBPRIME, then we both
+                    // prevented further updates here but also the (absent) oldval is
+                    // vaccuously available in the new table.  We return with true here:
+                    // any thread looking for a value for this key can correctly go
+                    // straight to the new table and skip looking in the old table.
+                    if (box == TOMBPRIME)
+                        return true;
+                    // Otherwise we boxed something, but it still needs to be
+                    // copied into the new table.
+                    oldval = box; // Record updated oldval
+                    break; // Break loop; oldval is now boxed by us
+                }
+                oldval = _vals[idx]; // Else try, try again
+            }
+            if (oldval == TOMBPRIME)
+                return false; // Copy already complete here!
+
+            // ---
+            // Copy the value into the new table, but only if we overwrite a null.
+            // If another value is already in the new table, then somebody else
+            // wrote something there and that write is happens-after any value that
+            // appears in the old table.  If putIfMatch does not find a null in the
+            // new table - somebody else should have recorded the null-not_null
+            // transition in this copy.
+            Object old_unboxed = ((Prime) oldval)._V;
+            assert old_unboxed != TOMBSTONE;
+            boolean copied_into_new = (_newchm.putIfMatch(key, old_unboxed, null) == null);
+
+            // ---
+            // Finally, now that any old value is exposed in the new table, we can
+            // forever hide the old-table value by slapping a TOMBPRIME down.  This
+            // will stop other threads from uselessly attempting to copy this slot
+            // (i.e., it's a speed optimization not a correctness issue).
+            while (!CAS_val(idx, oldval, TOMBPRIME))
+                oldval = _vals[idx];
+
+            return copied_into_new;
+        } // end copy_slot
+    } // End of CHM
+
+    // --- Snapshot ------------------------------------------------------------
+    private class SnapshotV implements Iterator, Enumeration {
+        final CHM _sschm;
+
+        public SnapshotV() {
+            CHM topchm;
+            while (true) { // Verify no table-copy-in-progress
+                topchm = _chm;
+                if (topchm._newchm == null) // No table-copy-in-progress
+                    break;
+                // Table copy in-progress - so we cannot get a clean iteration.  We
+                // must help finish the table copy before we can start iterating.
+                topchm.help_copy_impl(true);
+            }
+            // The "linearization point" for the iteration.  Every key in this table
+            // will be visited, but keys added later might be skipped or even be
+            // added to a following table (also not iterated over).
+            _sschm = topchm;
+            // Warm-up the iterator
+            _idx = -1;
+            next();
+        }
+
+        int length() {
+            return _sschm._keys.length;
+        }
+
+        long key(final int idx) {
+            return _sschm._keys[idx];
+        }
+
+        private int _idx; // -2 for NO_KEY, -1 for CHECK_NEW_TABLE_LONG, 0-keys.length
+        private long _nextK, _prevK; // Last 2 keys found
+        private TypeV _nextV, _prevV; // Last 2 values found
+
+        public boolean hasNext() {
+            return _nextV != null;
+        }
+
+        public TypeV next() {
+            // 'next' actually knows what the next value will be - it had to
+            // figure that out last go 'round lest 'hasNext' report true and
+            // some other thread deleted the last value.  Instead, 'next'
+            // spends all its effort finding the key that comes after the
+            // 'next' key.
+            if (_idx != -1 && _nextV == null)
+                throw new NoSuchElementException();
+            _prevK = _nextK; // This will become the previous key
+            _prevV = _nextV; // This will become the previous value
+            _nextV = null; // We have no more next-key
+            // Attempt to set <_nextK,_nextV> to the next K,V pair.
+            // _nextV is the trigger: stop searching when it is != null
+            if (_idx == -1) { // Check for NO_KEY
+                _idx = 0; // Setup for next phase of search
+                _nextK = NO_KEY;
+                if ((_nextV = get(_nextK)) != null)
+                    return _prevV;
+            }
+            while (_idx < length()) { // Scan array
+                _nextK = key(_idx++); // Get a key that definitely is in the set (for the moment!)
+                if (_nextK != NO_KEY && // Found something?
+                    (_nextV = get(_nextK)) != null)
+                    break; // Got it!  _nextK is a valid Key
+            } // Else keep scanning
+            return _prevV; // Return current value.
+        }
+
+        public void remove() {
+            if (_prevV == null)
+                throw new IllegalStateException();
+            _sschm.putIfMatch(_prevK, TOMBSTONE, _prevV);
+            _prevV = null;
+        }
+
+        public TypeV nextElement() {
+            return next();
+        }
+
+        public boolean hasMoreElements() {
+            return hasNext();
+        }
+    }
+
+    /**
+     * Returns an enumeration of the values in this table.
+     *
+     * @return an enumeration of the values in this table
+     * @see #values()
+     */
+    public Enumeration elements() {
+        return new SnapshotV();
+    }
+
+    // --- values --------------------------------------------------------------
+
+    /**
+     * Returns a {@link Collection} view of the values contained in this map.
+     * The collection is backed by the map, so changes to the map are reflected
+     * in the collection, and vice-versa.  The collection supports element
+     * removal, which removes the corresponding mapping from this map, via the
+     * Iterator.remove, Collection.remove,
+     * removeAll, retainAll, and clear operations.
+     * It does not support the add or addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator that
+     * will never throw {@link ConcurrentModificationException}, and guarantees
+     * to traverse elements as they existed upon construction of the iterator,
+     * and may (but is not guaranteed to) reflect any modifications subsequent
+     * to construction.
+     */
+    public Collection values() {
+        return new AbstractCollection() {
+            public void clear() {
+                NonBlockingHashMapLong.this.clear();
+            }
+
+            public int size() {
+                return NonBlockingHashMapLong.this.size();
+            }
+
+            public boolean contains(Object v) {
+                return NonBlockingHashMapLong.this.containsValue(v);
+            }
+
+            public Iterator iterator() {
+                return new SnapshotV();
+            }
+        };
+    }
+
+    // --- keySet --------------------------------------------------------------
+
+    /**
+     * A class which implements the {@link Iterator} and {@link Enumeration}
+     * interfaces, generified to the {@link Long} class and supporting a
+     * non-auto-boxing {@link #nextLong} function.
+     */
+    public class IteratorLong implements Iterator, Enumeration {
+        private final SnapshotV _ss;
+
+        /**
+         * A new IteratorLong
+         */
+        public IteratorLong() {
+            _ss = new SnapshotV();
+        }
+
+        /**
+         * Remove last key returned by {@link #next} or {@link #nextLong}.
+         */
+        public void remove() {
+            _ss.remove();
+        }
+
+        /**
+         * Auto-box and return the next key.
+         */
+        public Long next() {
+            _ss.next();
+            return _ss._prevK;
+        }
+
+        /**
+         * Return the next key as a primitive {@code long}.
+         */
+        public long nextLong() {
+            _ss.next();
+            return _ss._prevK;
+        }
+
+        /**
+         * True if there are more keys to iterate over.
+         */
+        public boolean hasNext() {
+            return _ss.hasNext();
+        }
+
+        /**
+         * Auto-box and return the next key.
+         */
+        public Long nextElement() {
+            return next();
+        }
+
+        /**
+         * True if there are more keys to iterate over.
+         */
+        public boolean hasMoreElements() {
+            return hasNext();
+        }
+    }
+
+    /**
+     * Returns an enumeration of the auto-boxed keys in this table.
+     * Warning: this version will auto-box all returned keys.
+     *
+     * @return an enumeration of the auto-boxed keys in this table
+     * @see #keySet()
+     */
+    public Enumeration keys() {
+        return new IteratorLong();
+    }
+
+    /**
+     * Returns a {@link Set} view of the keys contained in this map; with care
+     * the keys may be iterated over without auto-boxing.  The
+     * set is backed by the map, so changes to the map are reflected in the
+     * set, and vice-versa.  The set supports element removal, which removes
+     * the corresponding mapping from this map, via the
+     * Iterator.remove, Set.remove, removeAll,
+     * retainAll, and clear operations.  It does not support
+     * the add or addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator that
+     * will never throw {@link ConcurrentModificationException}, and guarantees
+     * to traverse elements as they existed upon construction of the iterator,
+     * and may (but is not guaranteed to) reflect any modifications subsequent
+     * to construction.
+     */
+    public Set keySet() {
+        return new AbstractSet() {
+            public void clear() {
+                NonBlockingHashMapLong.this.clear();
+            }
+
+            public int size() {
+                return NonBlockingHashMapLong.this.size();
+            }
+
+            public boolean contains(Object k) {
+                return NonBlockingHashMapLong.this.containsKey(k);
+            }
+
+            public boolean remove(Object k) {
+                return NonBlockingHashMapLong.this.remove(k) != null;
+            }
+
+            public IteratorLong iterator() {
+                return new IteratorLong();
+            }
+        };
+    }
+
+    /**
+     * Keys as a long array.  Array may be zero-padded if keys are concurrently deleted.
+     */
+    public long[] keySetLong() {
+        long[] dom = new long[size()];
+        IteratorLong i = (IteratorLong) keySet().iterator();
+        int j = 0;
+        while (j < dom.length && i.hasNext())
+            dom[j++] = i.nextLong();
+        return dom;
+    }
+
+    // --- entrySet ------------------------------------------------------------
+    // Warning: Each call to 'next' in this iterator constructs a new Long and a
+    // new NBHMLEntry.
+    private class NBHMLEntry extends AbstractEntry {
+        NBHMLEntry(final Long k, final TypeV v) {
+            super(k, v);
+        }
+
+        public TypeV setValue(final TypeV val) {
+            if (val == null)
+                throw new NullPointerException();
+            _val = val;
+            return put(_key, val);
+        }
+    }
+
+    private class SnapshotE implements Iterator> {
+        final SnapshotV _ss;
+
+        public SnapshotE() {
+            _ss = new SnapshotV();
+        }
+
+        public void remove() {
+            _ss.remove();
+        }
+
+        public Entry next() {
+            _ss.next();
+            return new NBHMLEntry(_ss._prevK, _ss._prevV);
+        }
+
+        public boolean hasNext() {
+            return _ss.hasNext();
+        }
+    }
+
+    /**
+     * Returns a {@link Set} view of the mappings contained in this map.  The
+     * set is backed by the map, so changes to the map are reflected in the
+     * set, and vice-versa.  The set supports element removal, which removes
+     * the corresponding mapping from the map, via the
+     * Iterator.remove, Set.remove, removeAll,
+     * retainAll, and clear operations.  It does not support
+     * the add or addAll operations.
+     * 
    
+     * 
    The view's iterator is a "weakly consistent" iterator
+     * that will never throw {@link ConcurrentModificationException},
+     * and guarantees to traverse elements as they existed upon
+     * construction of the iterator, and may (but is not guaranteed to)
+     * reflect any modifications subsequent to construction.
+     * 
    
+     */
+    public Set> entrySet() {
+        return new AbstractSet>() {
+            public void clear() {
+                NonBlockingHashMapLong.this.clear();
+            }
+
+            public int size() {
+                return NonBlockingHashMapLong.this.size();
+            }
+
+            public boolean remove(final Object o) {
+                if (!(o instanceof Map.Entry))
+                    return false;
+                final Entry e = (Entry) o;
+                return NonBlockingHashMapLong.this.remove(e.getKey(), e.getValue());
+            }
+
+            public boolean contains(final Object o) {
+                if (!(o instanceof Map.Entry))
+                    return false;
+                final Entry e = (Entry) o;
+                TypeV v = get(e.getKey());
+                return v.equals(e.getValue());
+            }
+
+            public Iterator> iterator() {
+                return new SnapshotE();
+            }
+        };
+    }
+
+    // --- writeObject -------------------------------------------------------
+    // Write a NBHML to a stream
+    private void writeObject(java.io.ObjectOutputStream s) throws IOException {
+        s.defaultWriteObject(); // Write nothing
+        for (long K : keySet()) {
+            final Object V = get(K); // Do an official 'get'
+            s.writeLong(K); // Write the  pair
+            s.writeObject(V);
+        }
+        s.writeLong(NO_KEY); // Sentinel to indicate end-of-data
+        s.writeObject(null);
+    }
+
+    // --- readObject --------------------------------------------------------
+    // Read a CHM from a stream
+    @SuppressWarnings("unchecked")
+    private void readObject(java.io.ObjectInputStream s) throws IOException, ClassNotFoundException {
+        s.defaultReadObject(); // Read nothing
+        initialize(MIN_SIZE);
+        for (;;) {
+            final long K = s.readLong();
+            final TypeV V = (TypeV) s.readObject();
+            if (K == NO_KEY && V == null)
+                break;
+            put(K, V); // Insert with an offical put
+        }
+    }
+
+} // End NonBlockingHashMapLong class
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/Dispatcher.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/Dispatcher.java
new file mode 100644
index 0000000..d31aea9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/Dispatcher.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+import java.util.concurrent.Executor;
+
+/**
+ * Task message dispatcher.
+ *
+ * @author jiachun.fjc
+ */
+public interface Dispatcher extends Executor {
+
+    /**
+     * Dispatch a task message.
+     */
+    boolean dispatch(final T message);
+
+    /**
+     * Shutdown
+     */
+    void shutdown();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/LoggingExceptionHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/LoggingExceptionHandler.java
new file mode 100644
index 0000000..32c66f1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/LoggingExceptionHandler.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.lmax.disruptor.ExceptionHandler;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class LoggingExceptionHandler implements ExceptionHandler {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LoggingExceptionHandler.class);
+
+    @Override
+    public void handleEventException(final Throwable ex, final long sequence, final Object event) {
+        if (LOG.isWarnEnabled()) {
+            LOG.warn("Exception processing: {} {}, {}.", sequence, event, StackTraceUtil.stackTrace(ex));
+        }
+    }
+
+    @Override
+    public void handleOnStartException(final Throwable ex) {
+        if (LOG.isWarnEnabled()) {
+            LOG.warn("Exception during onStart(), {}.", StackTraceUtil.stackTrace(ex));
+        }
+    }
+
+    @Override
+    public void handleOnShutdownException(final Throwable ex) {
+        if (LOG.isWarnEnabled()) {
+            LOG.warn("Exception during onShutdown(), {}.", StackTraceUtil.stackTrace(ex));
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/MessageEvent.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/MessageEvent.java
new file mode 100644
index 0000000..c72cb40
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/MessageEvent.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class MessageEvent {
+
+    private T message;
+
+    public T getMessage() {
+        return message;
+    }
+
+    public void reset() {
+        this.message = null;
+    }
+
+    public void setMessage(T message) {
+        this.message = message;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskDispatcher.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskDispatcher.java
new file mode 100644
index 0000000..64b3ec8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskDispatcher.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.util.Ints;
+import com.alipay.sofa.jraft.util.Requires;
+import com.lmax.disruptor.BlockingWaitStrategy;
+import com.lmax.disruptor.BusySpinWaitStrategy;
+import com.lmax.disruptor.EventFactory;
+import com.lmax.disruptor.InsufficientCapacityException;
+import com.lmax.disruptor.LiteBlockingWaitStrategy;
+import com.lmax.disruptor.LiteTimeoutBlockingWaitStrategy;
+import com.lmax.disruptor.PhasedBackoffWaitStrategy;
+import com.lmax.disruptor.RingBuffer;
+import com.lmax.disruptor.SleepingWaitStrategy;
+import com.lmax.disruptor.TimeoutBlockingWaitStrategy;
+import com.lmax.disruptor.WaitStrategy;
+import com.lmax.disruptor.YieldingWaitStrategy;
+import com.lmax.disruptor.dsl.Disruptor;
+import com.lmax.disruptor.dsl.ProducerType;
+
+/**
+ * The default wait strategy used by the Disruptor is the BlockingWaitStrategy.
+ *
+ * BlockingWaitStrategy:
+ * Internally the BlockingWaitStrategy uses a typical lock and condition variable to handle thread wake-up.
+ * The BlockingWaitStrategy is the slowest of the available wait strategies,
+ * but is the most conservative with the respect to CPU usage and will give the most consistent behaviour across
+ * the widest variety of deployment options. However, again knowledge of the deployed system can allow for additional
+ * performance.
+ *
+ * SleepingWaitStrategy:
+ * Like the BlockingWaitStrategy the SleepingWaitStrategy it attempts to be conservative with CPU usage,
+ * by using a simple busy wait loop, but uses a call to LockSupport.parkNanos(1) in the middle of the loop.
+ * On a typical Linux system this will pause the thread for around 60µs.
+ * However it has the benefit that the producing thread does not need to take any action other increment the appropriate
+ * counter and does not require the cost of signalling a condition variable. However, the mean latency of moving the
+ * event between the producer and consumer threads will be higher. It works best in situations where low latency is not
+ * required, but a low impact on the producing thread is desired. A common use case is for asynchronous logging.
+ *
+ * YieldingWaitStrategy:
+ * The YieldingWaitStrategy is one of 2 Wait Strategies that can be use in low latency systems,
+ * where there is the option to burn CPU cycles with the goal of improving latency.
+ * The YieldingWaitStrategy will busy spin waiting for the sequence to increment to the appropriate value.
+ * Inside the body of the loop Thread.yield() will be called allowing other queued threads to run.
+ * This is the recommended wait strategy when need very high performance and the number of Event Handler threads is
+ * less than the total number of logical cores, e.g. you have hyper-threading enabled.
+ *
+ * BusySpinWaitStrategy:
+ * The BusySpinWaitStrategy is the highest performing Wait Strategy, but puts the highest constraints on the deployment
+ * environment. This wait strategy should only be used if the number of Event Handler threads is smaller than the number
+ * of physical cores on the box. E.g. hyper-threading should be disabled
+ *
+ *
+ * @author jiachun.fjc
+ */
+public class TaskDispatcher implements Dispatcher {
+
+    private static final EventFactory> eventFactory = MessageEvent::new;
+
+    private final Disruptor> disruptor;
+
+    public TaskDispatcher(int bufSize, int numWorkers, WaitStrategyType waitStrategyType, ThreadFactory threadFactory) {
+        Requires.requireTrue(bufSize > 0, "bufSize must be larger than 0");
+        if (!Ints.isPowerOfTwo(bufSize)) {
+            bufSize = Ints.roundToPowerOfTwo(bufSize);
+        }
+        WaitStrategy waitStrategy;
+        switch (waitStrategyType) {
+            case BLOCKING_WAIT:
+                waitStrategy = new BlockingWaitStrategy();
+                break;
+            case LITE_BLOCKING_WAIT:
+                waitStrategy = new LiteBlockingWaitStrategy();
+                break;
+            case TIMEOUT_BLOCKING_WAIT:
+                waitStrategy = new TimeoutBlockingWaitStrategy(1000, TimeUnit.MILLISECONDS);
+                break;
+            case LITE_TIMEOUT_BLOCKING_WAIT:
+                waitStrategy = new LiteTimeoutBlockingWaitStrategy(1000, TimeUnit.MILLISECONDS);
+                break;
+            case PHASED_BACK_OFF_WAIT:
+                waitStrategy = PhasedBackoffWaitStrategy.withLiteLock(1000, 1000, TimeUnit.NANOSECONDS);
+                break;
+            case SLEEPING_WAIT:
+                waitStrategy = new SleepingWaitStrategy();
+                break;
+            case YIELDING_WAIT:
+                waitStrategy = new YieldingWaitStrategy();
+                break;
+            case BUSY_SPIN_WAIT:
+                waitStrategy = new BusySpinWaitStrategy();
+                break;
+            default:
+                throw new UnsupportedOperationException(waitStrategyType.toString());
+        }
+        this.disruptor = new Disruptor<>(eventFactory, bufSize, threadFactory, ProducerType.MULTI, waitStrategy);
+        this.disruptor.setDefaultExceptionHandler(new LoggingExceptionHandler());
+        if (numWorkers == 1) {
+            this.disruptor.handleEventsWith(new TaskHandler());
+        } else {
+            final TaskHandler[] handlers = new TaskHandler[numWorkers];
+            for (int i = 0; i < numWorkers; i++) {
+                handlers[i] = new TaskHandler();
+            }
+            this.disruptor.handleEventsWithWorkerPool(handlers);
+        }
+        this.disruptor.start();
+    }
+
+    @Override
+    public boolean dispatch(final Runnable message) {
+        final RingBuffer> ringBuffer = disruptor.getRingBuffer();
+        try {
+            final long sequence = ringBuffer.tryNext();
+            try {
+                final MessageEvent event = ringBuffer.get(sequence);
+                event.setMessage(message);
+            } finally {
+                ringBuffer.publish(sequence);
+            }
+            return true;
+        } catch (final InsufficientCapacityException e) {
+            // This exception is used by the Disruptor as a global goto. It is a singleton
+            // and has no stack trace.  Don't worry about performance.
+            return false;
+        }
+    }
+
+    @Override
+    public void execute(final Runnable message) {
+        if (!dispatch(message)) {
+            message.run(); // If fail to dispatch, caller run.
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        this.disruptor.shutdown();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskHandler.java
new file mode 100644
index 0000000..6fa51f7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/TaskHandler.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.lmax.disruptor.EventHandler;
+import com.lmax.disruptor.LifecycleAware;
+import com.lmax.disruptor.TimeoutHandler;
+import com.lmax.disruptor.WorkHandler;
+
+/**
+ * Callback interface to be implemented for processing events
+ * as they become available in the RingBuffer.
+ *
+ * @author jiachun.fjc
+ */
+public class TaskHandler implements EventHandler>, WorkHandler>,
+                        TimeoutHandler, LifecycleAware {
+
+    private static final Logger LOG = LoggerFactory.getLogger(TaskHandler.class);
+
+    @Override
+    public void onEvent(final MessageEvent event, final long sequence, final boolean endOfBatch)
+                                                                                                          throws Exception {
+        event.getMessage().run();
+        event.reset();
+    }
+
+    @Override
+    public void onEvent(final MessageEvent event) throws Exception {
+        event.getMessage().run();
+        event.reset();
+    }
+
+    @Override
+    public void onTimeout(final long sequence) throws Exception {
+        if (LOG.isWarnEnabled()) {
+            LOG.warn("Task timeout on: {}, sequence: {}.", Thread.currentThread().getName(), sequence);
+        }
+    }
+
+    @Override
+    public void onStart() {
+        if (LOG.isInfoEnabled()) {
+            LOG.info("Task handler on start: {}.", Thread.currentThread().getName());
+        }
+    }
+
+    @Override
+    public void onShutdown() {
+        if (LOG.isInfoEnabled()) {
+            LOG.info("Task handler on shutdown: {}.", Thread.currentThread().getName());
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/WaitStrategyType.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/WaitStrategyType.java
new file mode 100644
index 0000000..b807a87
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/concurrent/disruptor/WaitStrategyType.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.concurrent.disruptor;
+
+/**
+ * The default wait strategy used by the Disruptor is the BlockingWaitStrategy.
+ *
+ * BlockingWaitStrategy:
+ * Internally the BlockingWaitStrategy uses a typical lock and condition variable to handle thread wake-up.
+ * The BlockingWaitStrategy is the slowest of the available wait strategies,
+ * but is the most conservative with the respect to CPU usage and will give the most consistent behaviour across
+ * the widest variety of deployment options. However, again knowledge of the deployed system can allow for additional
+ * performance.
+ *
+ * SleepingWaitStrategy:
+ * Like the BlockingWaitStrategy the SleepingWaitStrategy it attempts to be conservative with CPU usage,
+ * by using a simple busy wait loop, but uses a call to LockSupport.parkNanos(1) in the middle of the loop.
+ * On a typical Linux system this will pause the thread for around 60µs.
+ * However it has the benefit that the producing thread does not need to take any action other increment the appropriate
+ * counter and does not require the cost of signalling a condition variable. However, the mean latency of moving the
+ * event between the producer and consumer threads will be higher. It works best in situations where low latency is not
+ * required, but a low impact on the producing thread is desired. A common use case is for asynchronous logging.
+ *
+ * YieldingWaitStrategy:
+ * The YieldingWaitStrategy is one of 2 Wait Strategies that can be use in low latency systems,
+ * where there is the option to burn CPU cycles with the goal of improving latency.
+ * The YieldingWaitStrategy will busy spin waiting for the sequence to increment to the appropriate value.
+ * Inside the body of the loop Thread.yield() will be called allowing other queued threads to run.
+ * This is the recommended wait strategy when need very high performance and the number of Event Handler threads is
+ * less than the total number of logical cores, e.g. you have hyper-threading enabled.
+ *
+ * BusySpinWaitStrategy:
+ * The BusySpinWaitStrategy is the highest performing Wait Strategy, but puts the highest constraints on the deployment
+ * environment. This wait strategy should only be used if the number of Event Handler threads is smaller than the number
+ * of physical cores on the box. E.g. hyper-threading should be disabled
+ *
+ * @author jiachun.fjc
+ */
+public enum WaitStrategyType {
+    BLOCKING_WAIT, //
+    LITE_BLOCKING_WAIT, //
+    TIMEOUT_BLOCKING_WAIT, //
+    LITE_TIMEOUT_BLOCKING_WAIT, //
+    PHASED_BACK_OFF_WAIT, //
+    SLEEPING_WAIT, //
+    YIELDING_WAIT, //
+    BUSY_SPIN_WAIT;
+
+    public static WaitStrategyType parse(String name) {
+        for (WaitStrategyType strategy : values()) {
+            if (strategy.name().equalsIgnoreCase(name)) {
+                return strategy;
+            }
+        }
+        return null;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/internal/UnsafeDirectBufferUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/internal/UnsafeDirectBufferUtil.java
new file mode 100644
index 0000000..f2cbdc9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/internal/UnsafeDirectBufferUtil.java
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.internal;
+
+import java.lang.reflect.Method;
+import java.nio.ByteOrder;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.util.internal.UnsafeUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class UnsafeDirectBufferUtil {
+
+    private static final Logger                    LOG                           = LoggerFactory
+                                                                                     .getLogger(UnsafeDirectBufferUtil.class);
+
+    private static final UnsafeUtil.UnsafeAccessor UNSAFE_ACCESSOR               = UnsafeUtil.getUnsafeAccessor();
+
+    private static final long                      BYTE_ARRAY_BASE_OFFSET        = UnsafeUtil
+                                                                                     .arrayBaseOffset(byte[].class);
+
+    // Limits the number of bytes to copy per {@link Unsafe#copyMemory(long, long, long)} to allow safepoint polling
+    // during a large copy.
+    private static final long                      UNSAFE_COPY_THRESHOLD         = 1024L * 1024L;
+
+    // These numbers represent the point at which we have empirically
+    // determined that the average cost of a JNI call exceeds the expense
+    // of an element by element copy.  These numbers may change over time.
+    private static final int                       JNI_COPY_TO_ARRAY_THRESHOLD   = 6;
+    private static final int                       JNI_COPY_FROM_ARRAY_THRESHOLD = 6;
+
+    private static final boolean                   BIG_ENDIAN_NATIVE_ORDER       = ByteOrder.nativeOrder() == ByteOrder.BIG_ENDIAN;
+    // Unaligned-access capability
+    private static final boolean                   UNALIGNED;
+
+    static {
+        boolean _unaligned;
+        try {
+            Class bitsClass = Class.forName("java.nio.Bits", false, UnsafeUtil.getSystemClassLoader());
+            Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned");
+            unalignedMethod.setAccessible(true);
+            _unaligned = (boolean) unalignedMethod.invoke(null);
+        } catch (Throwable t) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("java.nio.Bits: unavailable, {}.", StackTraceUtil.stackTrace(t));
+            }
+
+            _unaligned = false;
+        }
+        UNALIGNED = _unaligned;
+    }
+
+    public static byte getByte(long address) {
+        return UNSAFE_ACCESSOR.getByte(address);
+    }
+
+    public static short getShort(long address) {
+        if (UNALIGNED) {
+            short v = UNSAFE_ACCESSOR.getShort(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? v : Short.reverseBytes(v);
+        }
+        return (short) (UNSAFE_ACCESSOR.getByte(address) << 8 | UNSAFE_ACCESSOR.getByte(address + 1) & 0xff);
+    }
+
+    public static short getShortLE(long address) {
+        if (UNALIGNED) {
+            short v = UNSAFE_ACCESSOR.getShort(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? Short.reverseBytes(v) : v;
+        }
+        return (short) (UNSAFE_ACCESSOR.getByte(address) & 0xff | UNSAFE_ACCESSOR.getByte(address + 1) << 8);
+    }
+
+    public static int getInt(long address) {
+        if (UNALIGNED) {
+            int v = UNSAFE_ACCESSOR.getInt(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? v : Integer.reverseBytes(v);
+        }
+        return UNSAFE_ACCESSOR.getByte(address) << 24 //
+               | (UNSAFE_ACCESSOR.getByte(address + 1) & 0xff) << 16 //
+               | (UNSAFE_ACCESSOR.getByte(address + 2) & 0xff) << 8 //
+               | UNSAFE_ACCESSOR.getByte(address + 3) & 0xff;
+    }
+
+    public static int getIntLE(long address) {
+        if (UNALIGNED) {
+            int v = UNSAFE_ACCESSOR.getInt(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? Integer.reverseBytes(v) : v;
+        }
+        return UNSAFE_ACCESSOR.getByte(address) & 0xff //
+               | (UNSAFE_ACCESSOR.getByte(address + 1) & 0xff) << 8 //
+               | (UNSAFE_ACCESSOR.getByte(address + 2) & 0xff) << 16 //
+               | UNSAFE_ACCESSOR.getByte(address + 3) << 24;
+    }
+
+    public static long getLong(long address) {
+        if (UNALIGNED) {
+            long v = UNSAFE_ACCESSOR.getLong(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? v : Long.reverseBytes(v);
+        }
+        return ((long) UNSAFE_ACCESSOR.getByte(address)) << 56 //
+               | (UNSAFE_ACCESSOR.getByte(address + 1) & 0xffL) << 48 //
+               | (UNSAFE_ACCESSOR.getByte(address + 2) & 0xffL) << 40 //
+               | (UNSAFE_ACCESSOR.getByte(address + 3) & 0xffL) << 32 //
+               | (UNSAFE_ACCESSOR.getByte(address + 4) & 0xffL) << 24 //
+               | (UNSAFE_ACCESSOR.getByte(address + 5) & 0xffL) << 16 //
+               | (UNSAFE_ACCESSOR.getByte(address + 6) & 0xffL) << 8 //
+               | (UNSAFE_ACCESSOR.getByte(address + 7)) & 0xffL;
+    }
+
+    public static long getLongLE(long address) {
+        if (UNALIGNED) {
+            long v = UNSAFE_ACCESSOR.getLong(address);
+            return BIG_ENDIAN_NATIVE_ORDER ? Long.reverseBytes(v) : v;
+        }
+        return (UNSAFE_ACCESSOR.getByte(address)) & 0xffL //
+               | (UNSAFE_ACCESSOR.getByte(address + 1) & 0xffL) << 8 //
+               | (UNSAFE_ACCESSOR.getByte(address + 2) & 0xffL) << 16 //
+               | (UNSAFE_ACCESSOR.getByte(address + 3) & 0xffL) << 24 //
+               | (UNSAFE_ACCESSOR.getByte(address + 4) & 0xffL) << 32 //
+               | (UNSAFE_ACCESSOR.getByte(address + 5) & 0xffL) << 40 //
+               | (UNSAFE_ACCESSOR.getByte(address + 6) & 0xffL) << 48 //
+               | ((long) UNSAFE_ACCESSOR.getByte(address + 7)) << 56;
+    }
+
+    public static void getBytes(long address, byte[] dst, int dstIndex, int length) {
+        if (length > JNI_COPY_TO_ARRAY_THRESHOLD) {
+            copyMemory(null, address, dst, BYTE_ARRAY_BASE_OFFSET + dstIndex, length);
+        } else {
+            int end = dstIndex + length;
+            for (int i = dstIndex; i < end; i++) {
+                dst[i] = UNSAFE_ACCESSOR.getByte(address++);
+            }
+        }
+    }
+
+    public static void setByte(long address, int value) {
+        UNSAFE_ACCESSOR.putByte(address, (byte) value);
+    }
+
+    public static void setShort(long address, int value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putShort(address,
+                BIG_ENDIAN_NATIVE_ORDER ? (short) value : Short.reverseBytes((short) value));
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) (value >>> 8));
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) value);
+        }
+    }
+
+    public static void setShortLE(long address, int value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putShort(address, BIG_ENDIAN_NATIVE_ORDER ? Short.reverseBytes((short) value)
+                : (short) value);
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) value);
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) (value >>> 8));
+        }
+    }
+
+    public static void setInt(long address, int value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putInt(address, BIG_ENDIAN_NATIVE_ORDER ? value : Integer.reverseBytes(value));
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) (value >>> 24));
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) (value >>> 16));
+            UNSAFE_ACCESSOR.putByte(address + 2, (byte) (value >>> 8));
+            UNSAFE_ACCESSOR.putByte(address + 3, (byte) value);
+        }
+    }
+
+    public static void setIntLE(long address, int value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putInt(address, BIG_ENDIAN_NATIVE_ORDER ? Integer.reverseBytes(value) : value);
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) value);
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) (value >>> 8));
+            UNSAFE_ACCESSOR.putByte(address + 2, (byte) (value >>> 16));
+            UNSAFE_ACCESSOR.putByte(address + 3, (byte) (value >>> 24));
+        }
+    }
+
+    public static void setLong(long address, long value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putLong(address, BIG_ENDIAN_NATIVE_ORDER ? value : Long.reverseBytes(value));
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) (value >>> 56));
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) (value >>> 48));
+            UNSAFE_ACCESSOR.putByte(address + 2, (byte) (value >>> 40));
+            UNSAFE_ACCESSOR.putByte(address + 3, (byte) (value >>> 32));
+            UNSAFE_ACCESSOR.putByte(address + 4, (byte) (value >>> 24));
+            UNSAFE_ACCESSOR.putByte(address + 5, (byte) (value >>> 16));
+            UNSAFE_ACCESSOR.putByte(address + 6, (byte) (value >>> 8));
+            UNSAFE_ACCESSOR.putByte(address + 7, (byte) value);
+        }
+    }
+
+    public static void setLongLE(long address, long value) {
+        if (UNALIGNED) {
+            UNSAFE_ACCESSOR.putLong(address, BIG_ENDIAN_NATIVE_ORDER ? Long.reverseBytes(value) : value);
+        } else {
+            UNSAFE_ACCESSOR.putByte(address, (byte) value);
+            UNSAFE_ACCESSOR.putByte(address + 1, (byte) (value >>> 8));
+            UNSAFE_ACCESSOR.putByte(address + 2, (byte) (value >>> 16));
+            UNSAFE_ACCESSOR.putByte(address + 3, (byte) (value >>> 24));
+            UNSAFE_ACCESSOR.putByte(address + 4, (byte) (value >>> 32));
+            UNSAFE_ACCESSOR.putByte(address + 5, (byte) (value >>> 40));
+            UNSAFE_ACCESSOR.putByte(address + 6, (byte) (value >>> 48));
+            UNSAFE_ACCESSOR.putByte(address + 7, (byte) (value >>> 56));
+        }
+    }
+
+    public static void setBytes(long address, byte[] src, int srcIndex, int length) {
+        if (length > JNI_COPY_FROM_ARRAY_THRESHOLD) {
+            copyMemory(src, BYTE_ARRAY_BASE_OFFSET + srcIndex, null, address, length);
+        } else {
+            int end = srcIndex + length;
+            for (int i = srcIndex; i < end; i++) {
+                UNSAFE_ACCESSOR.putByte(address++, src[i]);
+            }
+        }
+    }
+
+    private static void copyMemory(Object src, long srcOffset, Object dst, long dstOffset, long length) {
+        while (length > 0) {
+            long size = Math.min(length, UNSAFE_COPY_THRESHOLD);
+            UNSAFE_ACCESSOR.copyMemory(src, srcOffset, dst, dstOffset, size);
+            length -= size;
+            srcOffset += size;
+            dstOffset += size;
+        }
+    }
+
+    private UnsafeDirectBufferUtil() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/AbstractHandlerContext.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/AbstractHandlerContext.java
new file mode 100644
index 0000000..e181145
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/AbstractHandlerContext.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+abstract class AbstractHandlerContext implements HandlerContext {
+
+    volatile AbstractHandlerContext next;
+    volatile AbstractHandlerContext prev;
+
+    private final boolean           inbound;
+    private final boolean           outbound;
+    private final DefaultPipeline   pipeline;
+    private final String            name;
+    private boolean                 removed;
+
+    final HandlerInvoker            invoker;
+
+    public AbstractHandlerContext(DefaultPipeline pipeline, HandlerInvoker invoker, String name, boolean inbound,
+                                  boolean outbound) {
+
+        if (name == null) {
+            throw new NullPointerException("name");
+        }
+
+        this.pipeline = pipeline;
+        if (invoker == null) {
+            this.invoker = new DefaultHandlerInvoker();
+        } else {
+            this.invoker = invoker;
+        }
+        this.name = name;
+
+        this.inbound = inbound;
+        this.outbound = outbound;
+    }
+
+    @Override
+    public Pipeline pipeline() {
+        return pipeline;
+    }
+
+    @Override
+    public HandlerInvoker invoker() {
+        return invoker;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public boolean isRemoved() {
+        return removed;
+    }
+
+    void setRemoved() {
+        removed = true;
+    }
+
+    @Override
+    public HandlerContext fireInbound(final InboundMessageEvent event) {
+        final AbstractHandlerContext next = findContextInbound();
+        next.invoker().invokeInbound(next, event);
+        return this;
+    }
+
+    @Override
+    public HandlerContext fireOutbound(final OutboundMessageEvent event) {
+        final AbstractHandlerContext next = findContextOutbound();
+        next.invoker().invokeOutbound(next, event);
+        return this;
+    }
+
+    @Override
+    public HandlerContext fireExceptionCaught(final MessageEvent event, final Throwable cause) {
+        final AbstractHandlerContext next = this.next;
+        next.invoker().invokeExceptionCaught(next, event, cause);
+        return this;
+    }
+
+    private AbstractHandlerContext findContextInbound() {
+        AbstractHandlerContext ctx = this;
+        do {
+            ctx = ctx.next;
+        } while (!ctx.inbound);
+        return ctx;
+    }
+
+    private AbstractHandlerContext findContextOutbound() {
+        AbstractHandlerContext ctx = this;
+        do {
+            ctx = ctx.prev;
+        } while (!ctx.outbound);
+        return ctx;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerContext.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerContext.java
new file mode 100644
index 0000000..f018022
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerContext.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+class DefaultHandlerContext extends AbstractHandlerContext {
+
+    private final Handler handler;
+
+    public DefaultHandlerContext(DefaultPipeline pipeline, HandlerInvoker invoker, String name, Handler handler) {
+        super(pipeline, invoker, name, isInbound(handler), isOutbound(handler));
+
+        this.handler = handler;
+    }
+
+    @Override
+    public Handler handler() {
+        return handler;
+    }
+
+    private static boolean isInbound(final Handler handler) {
+        return handler instanceof InboundHandler;
+    }
+
+    private static boolean isOutbound(final Handler handler) {
+        return handler instanceof OutboundHandler;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerInvoker.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerInvoker.java
new file mode 100644
index 0000000..ff79d7e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultHandlerInvoker.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.util.concurrent.ExecutorService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultHandlerInvoker implements HandlerInvoker {
+
+    private static final Logger   LOG = LoggerFactory.getLogger(DefaultHandlerInvoker.class);
+
+    private final ExecutorService executor;
+
+    public DefaultHandlerInvoker() {
+        this(null);
+    }
+
+    public DefaultHandlerInvoker(ExecutorService executor) {
+        this.executor = executor;
+    }
+
+    @Override
+    public ExecutorService executor() {
+        return executor;
+    }
+
+    @Override
+    public void invokeInbound(final HandlerContext ctx, final InboundMessageEvent event) {
+        if (this.executor == null) {
+            HandlerInvokerUtil.invokeInboundNow(ctx, event);
+        } else {
+            this.executor.execute(() -> HandlerInvokerUtil.invokeInboundNow(ctx, event));
+        }
+    }
+
+    @Override
+    public void invokeOutbound(final HandlerContext ctx, final OutboundMessageEvent event) {
+        if (this.executor == null) {
+            HandlerInvokerUtil.invokeOutboundNow(ctx, event);
+        } else {
+            this.executor.execute(() -> HandlerInvokerUtil.invokeOutboundNow(ctx, event));
+        }
+    }
+
+    @Override
+    public void invokeExceptionCaught(final HandlerContext ctx, final MessageEvent event, final Throwable cause) {
+        if (cause == null) {
+            throw new NullPointerException("cause");
+        }
+
+        if (this.executor == null) {
+            HandlerInvokerUtil.invokeExceptionCaughtNow(ctx, event, cause);
+        } else {
+            try {
+                this.executor.execute(() -> HandlerInvokerUtil.invokeExceptionCaughtNow(ctx, event, cause));
+            } catch (final Throwable t) {
+                if (LOG.isWarnEnabled()) {
+                    LOG.warn("Failed to submit an exceptionCaught() event: {}.", StackTraceUtil.stackTrace(t));
+                    LOG.warn("The exceptionCaught() event that was failed to submit was: {}.",
+                        StackTraceUtil.stackTrace(cause));
+                }
+            }
+        }
+    }
+
+    @Override
+    public void shutdown() {
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.executor);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultPipeline.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultPipeline.java
new file mode 100644
index 0000000..a5dbc81
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/DefaultPipeline.java
@@ -0,0 +1,650 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.WeakHashMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.future.DefaultPipelineFuture;
+import com.alipay.sofa.jraft.rhea.util.pipeline.future.PipelineFuture;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public final class DefaultPipeline implements Pipeline {
+
+    private static final Logger                             LOG        = LoggerFactory.getLogger(DefaultPipeline.class);
+
+    private static final ThreadLocal, String>> nameCaches = ThreadLocal.withInitial(WeakHashMap::new);
+
+    final AbstractHandlerContext                            head;
+    final AbstractHandlerContext                            tail;
+
+    public DefaultPipeline() {
+        tail = new TailContext(this);
+        head = new HeadContext(this);
+
+        head.next = tail;
+        tail.prev = head;
+    }
+
+    @Override
+    public Pipeline addFirst(String name, Handler handler) {
+        return addFirst(null, name, handler);
+    }
+
+    @Override
+    public Pipeline addFirst(HandlerInvoker invoker, String name, Handler handler) {
+        name = filterName(name, handler);
+        addFirst0(new DefaultHandlerContext(this, invoker, name, handler));
+        return this;
+    }
+
+    private void addFirst0(AbstractHandlerContext newCtx) {
+        checkMultiplicity(newCtx);
+
+        AbstractHandlerContext nextCtx = head.next;
+        newCtx.prev = head;
+        newCtx.next = nextCtx;
+        head.next = newCtx;
+        nextCtx.prev = newCtx;
+
+        callHandlerAdded(newCtx);
+    }
+
+    @Override
+    public Pipeline addLast(String name, Handler handler) {
+        return addLast(null, name, handler);
+    }
+
+    @Override
+    public Pipeline addLast(HandlerInvoker invoker, String name, Handler handler) {
+        name = filterName(name, handler);
+        addLast0(new DefaultHandlerContext(this, invoker, name, handler));
+        return this;
+    }
+
+    private void addLast0(AbstractHandlerContext newCtx) {
+        checkMultiplicity(newCtx);
+
+        AbstractHandlerContext prev = tail.prev;
+        newCtx.prev = prev;
+        newCtx.next = tail;
+        prev.next = newCtx;
+        tail.prev = newCtx;
+
+        callHandlerAdded(newCtx);
+    }
+
+    @Override
+    public Pipeline addBefore(String baseName, String name, Handler handler) {
+        return addBefore(null, baseName, name, handler);
+    }
+
+    @Override
+    public Pipeline addBefore(HandlerInvoker invoker, String baseName, String name, Handler handler) {
+        AbstractHandlerContext ctx = getContextOrDie(baseName);
+        name = filterName(name, handler);
+        addBefore0(ctx, new DefaultHandlerContext(this, invoker, name, handler));
+        return this;
+    }
+
+    private void addBefore0(AbstractHandlerContext ctx, AbstractHandlerContext newCtx) {
+        checkMultiplicity(newCtx);
+
+        newCtx.prev = ctx.prev;
+        newCtx.next = ctx;
+        ctx.prev.next = newCtx;
+        ctx.prev = newCtx;
+
+        callHandlerAdded(newCtx);
+    }
+
+    @Override
+    public Pipeline addAfter(String baseName, String name, Handler handler) {
+        return addAfter(null, baseName, name, handler);
+    }
+
+    @Override
+    public Pipeline addAfter(HandlerInvoker invoker, String baseName, String name, Handler handler) {
+        AbstractHandlerContext ctx = getContextOrDie(baseName);
+        name = filterName(name, handler);
+        addAfter0(ctx, new DefaultHandlerContext(this, invoker, name, handler));
+        return this;
+    }
+
+    private void addAfter0(AbstractHandlerContext ctx, AbstractHandlerContext newCtx) {
+        checkMultiplicity(newCtx);
+
+        newCtx.prev = ctx;
+        newCtx.next = ctx.next;
+        ctx.next.prev = newCtx;
+        ctx.next = newCtx;
+
+        callHandlerAdded(newCtx);
+    }
+
+    @Override
+    public Pipeline addFirst(Handler... handlers) {
+        return addFirst(null, handlers);
+    }
+
+    @Override
+    public Pipeline addFirst(HandlerInvoker invoker, Handler... handlers) {
+        if (handlers == null) {
+            throw new NullPointerException("handlers");
+        }
+        if (handlers.length == 0 || handlers[0] == null) {
+            return this;
+        }
+
+        int size;
+        for (size = 1; size < handlers.length; size ++) {
+            if (handlers[size] == null) {
+                break;
+            }
+        }
+
+        for (int i = size - 1; i >= 0; i --) {
+            Handler h = handlers[i];
+            addFirst(invoker, null, h);
+        }
+
+        return this;
+    }
+
+    @Override
+    public Pipeline addLast(Handler... handlers) {
+        return addLast(null, handlers);
+    }
+
+    @Override
+    public Pipeline addLast(HandlerInvoker invoker, Handler... handlers) {
+        if (handlers == null) {
+            throw new NullPointerException("handlers");
+        }
+
+        for (Handler h: handlers) {
+            if (h == null) {
+                break;
+            }
+            addLast(invoker, null, h);
+        }
+
+        return this;
+    }
+
+    @Override
+    public Pipeline remove(Handler handler) {
+        remove(getContextOrDie(handler));
+        return this;
+    }
+
+    @Override
+    public Handler remove(String name) {
+        return remove(getContextOrDie(name)).handler();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  T remove(Class handlerType) {
+        return (T) remove(getContextOrDie(handlerType)).handler();
+    }
+
+    private AbstractHandlerContext remove(final AbstractHandlerContext ctx) {
+        assert ctx != head && ctx != tail;
+
+        synchronized (this) {
+            remove0(ctx);
+            return ctx;
+        }
+    }
+
+    private void remove0(AbstractHandlerContext ctx) {
+        AbstractHandlerContext prev = ctx.prev;
+        AbstractHandlerContext next = ctx.next;
+        prev.next = next;
+        next.prev = prev;
+        callHandlerRemoved(ctx);
+    }
+
+    @Override
+    public Handler removeFirst() {
+        if (head.next == tail) {
+            throw new NoSuchElementException();
+        }
+        return remove(head.next).handler();
+    }
+
+    @Override
+    public Handler removeLast() {
+        if (head.next == tail) {
+            throw new NoSuchElementException();
+        }
+        return remove(tail.prev).handler();
+    }
+
+    @Override
+    public Pipeline replace(Handler oldHandler, String newName, Handler newHandler) {
+        replace(getContextOrDie(oldHandler), newName, newHandler);
+        return this;
+    }
+
+    @Override
+    public Handler replace(String oldName, String newName, Handler newHandler) {
+        return replace(getContextOrDie(oldName), newName, newHandler);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  T replace(Class oldHandlerType, String newName, Handler newHandler) {
+        return (T) replace(getContextOrDie(oldHandlerType), newName, newHandler);
+    }
+
+    private Handler replace(
+            final AbstractHandlerContext ctx, String newName, Handler newHandler) {
+
+        assert ctx != head && ctx != tail;
+
+        synchronized (this) {
+            if (newName == null) {
+                newName = ctx.name();
+            } else if (!ctx.name().equals(newName)) {
+                newName = filterName(newName, newHandler);
+            }
+
+            final AbstractHandlerContext newCtx =
+                    new DefaultHandlerContext(this, ctx.invoker, newName, newHandler);
+
+            replace0(ctx, newCtx);
+            return ctx.handler();
+        }
+    }
+
+    private void replace0(AbstractHandlerContext oldCtx, AbstractHandlerContext newCtx) {
+        checkMultiplicity(newCtx);
+
+        AbstractHandlerContext prev = oldCtx.prev;
+        AbstractHandlerContext next = oldCtx.next;
+        newCtx.prev = prev;
+        newCtx.next = next;
+
+        // Finish the replacement of oldCtx with newCtx in the linked list.
+        // Note that this doesn't mean events will be sent to the new handler immediately
+        // because we are currently at the event handler thread and no more than one handler methods can be invoked
+        // at the same time (we ensured that in replace().)
+        prev.next = newCtx;
+        next.prev = newCtx;
+
+        // update the reference to the replacement so forward of buffered content will work correctly
+        oldCtx.prev = newCtx;
+        oldCtx.next = newCtx;
+
+        // Invoke newHandler.handlerAdded() first (i.e. before oldHandler.handlerRemoved() is invoked)
+        // because callHandlerRemoved() will trigger inboundBufferUpdated() or flush() on newHandler and those
+        // event handlers must be called after handlerAdded().
+        callHandlerAdded(newCtx);
+        callHandlerRemoved(oldCtx);
+    }
+
+    @Override
+    public Handler get(String name) {
+        HandlerContext ctx = context(name);
+        if (ctx == null) {
+            return null;
+        } else {
+            return ctx.handler();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public  T get(Class handlerType) {
+        HandlerContext ctx = context(handlerType);
+        if (ctx == null) {
+            return null;
+        } else {
+            return (T) ctx.handler();
+        }
+    }
+
+    @Override
+    public HandlerContext context(Handler handler) {
+        if (handler == null) {
+            throw new NullPointerException("handler");
+        }
+
+        AbstractHandlerContext ctx = head.next;
+        for (;;) {
+
+            if (ctx == null) {
+                return null;
+            }
+
+            if (ctx.handler() == handler) {
+                return ctx;
+            }
+
+            ctx = ctx.next;
+        }
+    }
+
+    @Override
+    public HandlerContext context(String name) {
+        if (name == null) {
+            throw new NullPointerException("name");
+        }
+
+        return context0(name);
+    }
+
+    @Override
+    public HandlerContext context(Class handlerType) {
+        if (handlerType == null) {
+            throw new NullPointerException("handlerType");
+        }
+
+        AbstractHandlerContext ctx = head.next;
+        for (;;) {
+            if (ctx == null) {
+                return null;
+            }
+            if (handlerType.isAssignableFrom(ctx.handler().getClass())) {
+                return ctx;
+            }
+            ctx = ctx.next;
+        }
+    }
+
+    @Override
+    public Pipeline fireInbound(InboundMessageEvent event) {
+        head.fireInbound(event);
+        return this;
+    }
+
+    @Override
+    public Pipeline fireOutbound(OutboundMessageEvent event) {
+        tail.fireOutbound(event);
+        return this;
+    }
+
+    @Override
+    public Pipeline fireExceptionCaught(MessageEvent event, Throwable cause) {
+        head.fireExceptionCaught(event, cause);
+        return this;
+    }
+
+    @Override
+    public  PipelineFuture invoke(InboundMessageEvent event) {
+        return invoke(event, -1);
+    }
+
+    @Override
+    public  PipelineFuture invoke(InboundMessageEvent event, long timeoutMillis) {
+        PipelineFuture future = DefaultPipelineFuture.with(event.getInvokeId(), timeoutMillis);
+        head.fireInbound(event);
+        return future;
+    }
+
+    private void callHandlerAdded(final AbstractHandlerContext ctx) {
+        try {
+            ctx.handler().handlerAdded(ctx);
+        } catch (Throwable t) {
+            boolean removed = false;
+            try {
+                remove(ctx);
+                removed = true;
+            } catch (Throwable t2) {
+                if (LOG.isWarnEnabled()) {
+                    LOG.warn("Failed to remove a handler: {}, {}.", ctx.name(), StackTraceUtil.stackTrace(t2));
+                }
+            }
+
+        fireExceptionCaught(null, new PipelineException(
+                ctx.handler().getClass().getName() +
+                        ".handlerAdded() has thrown an exception; " + (removed ? "removed." :  "also failed to remove."), t));
+        }
+    }
+
+    private void callHandlerRemoved(final AbstractHandlerContext ctx) {
+        // Notify the complete removal.
+        try {
+            ctx.handler().handlerRemoved(ctx);
+            ctx.setRemoved();
+        } catch (Throwable t) {
+            fireExceptionCaught(null, new PipelineException(
+                    ctx.handler().getClass().getName() + ".handlerRemoved() has thrown an exception.", t));
+        }
+    }
+
+    private static void checkMultiplicity(HandlerContext ctx) {
+        Handler handler = ctx.handler();
+        if (handler instanceof HandlerAdapter) {
+            HandlerAdapter h = (HandlerAdapter) handler;
+            if (!h.isSharable() && h.added) {
+                throw new PipelineException(
+                        h.getClass().getName() +
+                                " is not a @Sharable handler, so can't be added or removed multiple times.");
+            }
+            h.added = true;
+        }
+    }
+
+    private String filterName(String name, Handler handler) {
+        if (name == null) {
+            return generateName(handler);
+        }
+
+        if (context0(name) == null) {
+            return name;
+        }
+
+        throw new IllegalArgumentException("Duplicate handler name: " + name);
+    }
+
+    private AbstractHandlerContext context0(String name) {
+        AbstractHandlerContext context = head.next;
+        while (context != tail) {
+            if (context.name().equals(name)) {
+                return context;
+            }
+            context = context.next;
+        }
+        return null;
+    }
+
+    private AbstractHandlerContext getContextOrDie(String name) {
+        AbstractHandlerContext ctx = (AbstractHandlerContext) context(name);
+        if (ctx == null) {
+            throw new NoSuchElementException(name);
+        } else {
+            return ctx;
+        }
+    }
+
+    private AbstractHandlerContext getContextOrDie(Handler handler) {
+        AbstractHandlerContext ctx = (AbstractHandlerContext) context(handler);
+        if (ctx == null) {
+            throw new NoSuchElementException(handler.getClass().getName());
+        } else {
+            return ctx;
+        }
+    }
+
+    private AbstractHandlerContext getContextOrDie(Class handlerType) {
+        AbstractHandlerContext ctx = (AbstractHandlerContext) context(handlerType);
+        if (ctx == null) {
+            throw new NoSuchElementException(handlerType.getName());
+        } else {
+            return ctx;
+        }
+    }
+
+    private String generateName(Handler handler) {
+        Map, String> cache = nameCaches.get();
+        Class handlerType = handler.getClass();
+        String name = cache.get(handlerType);
+        if (name == null) {
+            name = generateName0(handlerType);
+            cache.put(handlerType, name);
+        }
+
+        synchronized (this) {
+            // It's not very likely for a user to put more than one handler of the same type, but make sure to avoid
+            // any name conflicts.  Note that we don't cache the names generated here.
+            if (context0(name) != null) {
+                String baseName = name.substring(0, name.length() - 1); // Strip the trailing '0'.
+                for (int i = 1;; i ++) {
+                    String newName = baseName + i;
+                    if (context0(newName) == null) {
+                        name = newName;
+                        break;
+                    }
+                }
+            }
+        }
+
+        return name;
+    }
+
+    private static String generateName0(Class handlerType) {
+        if (handlerType == null) {
+            throw new NullPointerException("handlerType");
+        }
+
+        String className = handlerType.getName();
+        final int lastDotIdx = className.lastIndexOf('.');
+        if (lastDotIdx > -1) {
+            className = className.substring(lastDotIdx + 1);
+        }
+        return className + "#0";
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder buf = new StringBuilder()
+                .append(getClass().getSimpleName())
+                .append('{');
+        AbstractHandlerContext ctx = head.next;
+        for (;;) {
+            if (ctx == tail) {
+                break;
+            }
+
+            buf.append('(')
+                    .append(ctx.name())
+                    .append(" = ")
+                    .append(ctx.handler().getClass().getName())
+                    .append(')');
+
+            ctx = ctx.next;
+            if (ctx == tail) {
+                break;
+            }
+
+            buf.append(", ");
+        }
+        buf.append('}');
+        return buf.toString();
+    }
+
+    // A special catch-all handler that handles both messages.
+    static final class TailContext extends AbstractHandlerContext implements InboundHandler {
+
+        private static final String TAIL_NAME = generateName0(TailContext.class);
+
+        public TailContext(DefaultPipeline pipeline) {
+            super(pipeline, null, TAIL_NAME, true, false);
+        }
+
+        @Override
+        public boolean isAcceptable(MessageEvent event) {
+            return true;
+        }
+
+        @Override
+        public void handleInbound(HandlerContext ctx, InboundMessageEvent event) throws Exception {}
+
+        @Override
+        public void exceptionCaught(HandlerContext ctx, MessageEvent event, Throwable cause) throws Exception {
+            LOG.warn("An exceptionCaught() event was fired, {}.", StackTraceUtil.stackTrace(cause));
+
+            if (event != null) {
+                DefaultPipelineFuture.received(event.getInvokeId(), cause);
+            }
+        }
+
+        @Override
+        public void handlerAdded(HandlerContext ctx) throws Exception {}
+
+        @Override
+        public void handlerRemoved(HandlerContext ctx) throws Exception {}
+
+        @Override
+        public Handler handler() {
+            return this;
+        }
+    }
+
+    static final class HeadContext extends AbstractHandlerContext implements OutboundHandler {
+
+        private static final String HEAD_NAME = generateName0(HeadContext.class);
+
+        public HeadContext(DefaultPipeline pipeline) {
+            super(pipeline, null, HEAD_NAME, false, true);
+        }
+
+        @Override
+        public boolean isAcceptable(MessageEvent event) {
+            return true;
+        }
+
+        @Override
+        public void handlerAdded(HandlerContext ctx) throws Exception {}
+
+        @Override
+        public void handlerRemoved(HandlerContext ctx) throws Exception {}
+
+        @Override
+        public void exceptionCaught(HandlerContext ctx, MessageEvent event, Throwable cause) throws Exception {
+            ctx.fireExceptionCaught(event, cause);
+        }
+
+        @Override
+        public Handler handler() {
+            return this;
+        }
+
+        @Override
+        public void handleOutbound(HandlerContext ctx, OutboundMessageEvent event) throws Exception {
+            if (event != null) {
+                DefaultPipelineFuture.received(event.getInvokeId(), event.getMessage());
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Handler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Handler.java
new file mode 100644
index 0000000..4e1b48a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Handler.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface Handler {
+
+    boolean isAcceptable(final MessageEvent event);
+
+    /**
+     * Gets called after the {@link Handler} was added to the actual context and it's ready to handle events.
+     */
+    void handlerAdded(final HandlerContext ctx) throws Exception;
+
+    /**
+     * Gets called after the {@link Handler} was removed from the actual context and it doesn't handle events anymore.
+     */
+    void handlerRemoved(final HandlerContext ctx) throws Exception;
+
+    /**
+     * Gets called if a {@link Throwable} was thrown.
+     */
+    void exceptionCaught(final HandlerContext ctx, final MessageEvent event, final Throwable cause) throws Exception;
+
+    /**
+     * Indicates that the same instance of the annotated {@link Handler}
+     * can be added to one or more {@link Pipeline}s multiple times
+     * without a race condition.
+     *
+     * If this annotation is not specified, you have to create a new handler
+     * instance every time you add it to a pipeline because it has unshared
+     * state such as member variables.
+     */
+    @Inherited
+    @Documented
+    @Target(ElementType.TYPE)
+    @Retention(RetentionPolicy.RUNTIME)
+    @interface Sharable {
+        // no value
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerAdapter.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerAdapter.java
new file mode 100644
index 0000000..43cb5e3
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerAdapter.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.util.concurrent.ConcurrentMap;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class HandlerAdapter implements Handler {
+
+    // Not using volatile because it's used only for a sanity check.
+    boolean                                               added;
+
+    private static final ConcurrentMap, Boolean> cache = Maps.newConcurrentMap();
+
+    /**
+     * Do nothing by default, sub-classes may override this method.
+     */
+    @Override
+    public void handlerAdded(final HandlerContext ctx) throws Exception {
+        // NOOP
+    }
+
+    /**
+     * Do nothing by default, sub-classes may override this method.
+     */
+    @Override
+    public void handlerRemoved(final HandlerContext ctx) throws Exception {
+        // NOOP
+    }
+
+    /**
+     * Return {@code true} if the implementation is {@link Sharable} and so can be added
+     * to different {@link Pipeline}s.
+     */
+    public boolean isSharable() {
+        final Class clazz = getClass();
+        Boolean sharable = cache.get(clazz);
+        if (sharable == null) {
+            sharable = clazz.isAnnotationPresent(Sharable.class);
+            cache.put(clazz, sharable);
+        }
+        return sharable;
+    }
+
+    /**
+     * Calls {@link HandlerContext#fireExceptionCaught(MessageEvent, Throwable)} to forward
+     * to the next {@link Handler} in the {@link Pipeline}.
+     *
+     * Sub-classes may override this method to change behavior.
+     */
+    @Override
+    public void exceptionCaught(final HandlerContext ctx, final MessageEvent event, final Throwable cause)
+                                                                                                             throws Exception {
+        ctx.fireExceptionCaught(event, cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerContext.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerContext.java
new file mode 100644
index 0000000..469f680
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerContext.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Enables a {@link Handler} to interact with its {@link Pipeline}
+ * and other handlers. Among other things a handler can notify the next {@link Handler} in the
+ * {@link Pipeline} as well as modify the {@link Pipeline} it belongs to dynamically.
+ *
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface HandlerContext {
+
+    /**
+     * Return the assigned {@link Pipeline}.
+     */
+    Pipeline pipeline();
+
+    /**
+     * Returns the {@link HandlerInvoker} which is used to trigger an event for the associated
+     * {@link Handler}. Note that the methods in {@link HandlerInvoker} are not intended to be called
+     * by a user. Use this method only to obtain the reference to the {@link HandlerInvoker}
+     * (and not calling its methods) unless you know what you are doing.
+     */
+    HandlerInvoker invoker();
+
+    /**
+     * The unique name of the {@link HandlerContext}.The name was used when then {@link Handler}
+     * was added to the {@link Pipeline}. This name can also be used to access the registered
+     * {@link Handler} from the {@link Pipeline}.
+     */
+    String name();
+
+    /**
+     * The {@link Handler} that is bound this {@link HandlerContext}.
+     */
+    Handler handler();
+
+    /**
+     * Return {@code true} if the {@link Handler} which belongs to this context was removed
+     * from the {@link Pipeline}.
+     */
+    boolean isRemoved();
+
+    /**
+     * Received a message.
+     */
+    HandlerContext fireInbound(final InboundMessageEvent event);
+
+    /**
+     * Request to write a message via this {@link HandlerContext} through the {@link Pipeline}.
+     */
+    HandlerContext fireOutbound(final OutboundMessageEvent event);
+
+    /**
+     * Received an {@link Throwable} in one of its inbound operations.
+     */
+    HandlerContext fireExceptionCaught(final MessageEvent event, final Throwable cause);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvoker.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvoker.java
new file mode 100644
index 0000000..2d538e6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvoker.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface HandlerInvoker {
+
+    /**
+     * Returns the {@link Executor} which is used to execute an arbitrary task.
+     */
+    ExecutorService executor();
+
+    /**
+     * Invokes {@link InboundHandler#handleInbound(HandlerContext, InboundMessageEvent)}. This method is not for a user
+     * but for the internal {@link HandlerContext} implementation. To trigger an event, use the methods in
+     * {@link HandlerContext} instead.
+     */
+    void invokeInbound(final HandlerContext ctx, final InboundMessageEvent event);
+
+    /**
+     * Invokes {@link OutboundHandler#handleOutbound(HandlerContext, OutboundMessageEvent)}. This method is not for a user
+     * but for the internal {@link HandlerContext} implementation. To trigger an event, use the methods in
+     * {@link HandlerContext} instead.
+     */
+    void invokeOutbound(final HandlerContext ctx, final OutboundMessageEvent event);
+
+    /**
+     * Received an {@link Throwable} in one of its inbound operations.
+     *
+     * This will result in having the {@link InboundHandler#exceptionCaught(HandlerContext, MessageEvent, Throwable)}
+     * method called of the next {@link InboundHandler} contained in the {@link Pipeline}.
+     */
+    void invokeExceptionCaught(final HandlerContext ctx, final MessageEvent event, final Throwable cause);
+
+    /**
+     * Attempts to stop all actively executing tasks.
+     */
+    void shutdown();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvokerUtil.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvokerUtil.java
new file mode 100644
index 0000000..9a52886
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/HandlerInvokerUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public final class HandlerInvokerUtil {
+
+    private static final Logger LOG = LoggerFactory.getLogger(HandlerInvokerUtil.class);
+
+    public static void invokeInboundNow(final HandlerContext ctx, final InboundMessageEvent event) {
+        try {
+            ((InboundHandler) ctx.handler()).handleInbound(ctx, event);
+        } catch (final Throwable t) {
+            notifyHandlerException(ctx, event, t);
+        }
+    }
+
+    public static void invokeOutboundNow(final HandlerContext ctx, final OutboundMessageEvent event) {
+        try {
+            ((OutboundHandler) ctx.handler()).handleOutbound(ctx, event);
+        } catch (final Throwable t) {
+            notifyHandlerException(ctx, event, t);
+        }
+    }
+
+    public static void invokeExceptionCaughtNow(final HandlerContext ctx, final MessageEvent event,
+                                                final Throwable cause) {
+        try {
+            ctx.handler().exceptionCaught(ctx, event, cause);
+        } catch (final Throwable t) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("An exception was thrown by a user handler's exceptionCaught() method: {}.",
+                    StackTraceUtil.stackTrace(t));
+                LOG.warn(".. and the cause of the exceptionCaught() was: {}.", StackTraceUtil.stackTrace(cause));
+            }
+        }
+    }
+
+    private static void notifyHandlerException(final HandlerContext ctx, final MessageEvent event,
+                                               final Throwable cause) {
+        if (inExceptionCaught(cause)) {
+            if (LOG.isWarnEnabled()) {
+                LOG.warn("An exception was thrown by a user handler " + "while handling an exceptionCaught event: {}.",
+                    StackTraceUtil.stackTrace(cause));
+            }
+            return;
+        }
+
+        invokeExceptionCaughtNow(ctx, event, cause);
+    }
+
+    private static boolean inExceptionCaught(Throwable cause) {
+        do {
+            final StackTraceElement[] trace = cause.getStackTrace();
+            if (trace != null) {
+                for (final StackTraceElement t : trace) {
+                    if (t == null) {
+                        break;
+                    }
+                    if ("exceptionCaught".equals(t.getMethodName())) {
+                        return true;
+                    }
+                }
+            }
+
+            cause = cause.getCause();
+        } while (cause != null);
+
+        return false;
+    }
+
+    private HandlerInvokerUtil() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandler.java
new file mode 100644
index 0000000..818d02e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface InboundHandler extends Handler {
+
+    /**
+     * Handles the specified inbound event.
+     *
+     * @param ctx   the context object for this handler.
+     * @param event the inbound event to process or intercept.
+     */
+    void handleInbound(final HandlerContext ctx, final InboundMessageEvent event) throws Exception;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandlerAdapter.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandlerAdapter.java
new file mode 100644
index 0000000..370ae2d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/InboundHandlerAdapter.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class InboundHandlerAdapter extends HandlerAdapter implements InboundHandler {
+
+    private final TypeParameterMatcher matcher;
+
+    protected InboundHandlerAdapter() {
+        this.matcher = TypeParameterMatcher.find(this, InboundHandlerAdapter.class, "I");
+    }
+
+    @Override
+    public boolean isAcceptable(final MessageEvent event) {
+        return matcher.match(event);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void handleInbound(final HandlerContext ctx, final InboundMessageEvent event) throws Exception {
+        if (isAcceptable(event)) {
+            readMessage(ctx, (I) event);
+        }
+        ctx.fireInbound(event);
+    }
+
+    /**
+     * If you expect that the event to continue to flow in the pipeline,
+     * you should to call {@code ctx.fireInbound(event)} or {@code ctx.fireOutbound(event)}
+     * at the end of the method.
+     */
+    public abstract void readMessage(final HandlerContext ctx, final I event) throws Exception;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandler.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandler.java
new file mode 100644
index 0000000..a0ef158
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface OutboundHandler extends Handler {
+
+    /**
+     * Handles the specified outbound event.
+     *
+     * @param ctx   the context object for this handler.
+     * @param event the outbound event to process or intercept.
+     */
+    void handleOutbound(final HandlerContext ctx, final OutboundMessageEvent event) throws Exception;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandlerAdapter.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandlerAdapter.java
new file mode 100644
index 0000000..b12feca
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/OutboundHandlerAdapter.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class OutboundHandlerAdapter extends HandlerAdapter implements OutboundHandler {
+
+    private final TypeParameterMatcher matcher;
+
+    protected OutboundHandlerAdapter() {
+        this.matcher = TypeParameterMatcher.find(this, OutboundHandlerAdapter.class, "I");
+    }
+
+    @Override
+    public boolean isAcceptable(final MessageEvent event) {
+        return matcher.match(event);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public void handleOutbound(final HandlerContext ctx, final OutboundMessageEvent event) throws Exception {
+        if (isAcceptable(event)) {
+            writeMessage(ctx, (I) event);
+        }
+        ctx.fireOutbound(event);
+    }
+
+    /**
+     * If you expect that the event to continue to flow in the pipeline,
+     * you should to call {@code ctx.fireOutbound(event)} at the end of the method.
+     */
+    public abstract void writeMessage(final HandlerContext ctx, final I event) throws Exception;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Pipeline.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Pipeline.java
new file mode 100644
index 0000000..d89ec53
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/Pipeline.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.MessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.future.PipelineFuture;
+
+/**
+ * A list of {@link Handler}s which handles or intercepts
+ * inbound events and outbound operations.
+ *
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public interface Pipeline {
+
+    /**
+     * Inserts a {@link Handler} at the first position of this pipeline.
+     *
+     * @param name      the name of the handler to insert first.
+     * @param handler   the handler to insert first.
+     *
+     * @return itself
+     */
+    Pipeline addFirst(final String name, final Handler handler);
+
+    /**
+     * Inserts a {@link Handler} at the first position of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}'s methods.
+     * @param name      the name of the handler to insert first.
+     * @param handler   the handler to insert first.
+     *
+     * @return itself
+     */
+    Pipeline addFirst(final HandlerInvoker invoker, final String name, final Handler handler);
+
+    /**
+     * Appends a {@link Handler} at the last position of this pipeline.
+     *
+     * @param name      the name of the handler to append.
+     * @param handler   the handler to append.
+     *
+     * @return itself
+     */
+    Pipeline addLast(final String name, final Handler handler);
+
+    /**
+     * Appends a {@link Handler} at the last position of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}'s methods.
+     * @param name      the name of the handler to append.
+     * @param handler   the handler to append.
+     *
+     * @return itself
+     */
+    Pipeline addLast(final HandlerInvoker invoker, final String name, final Handler handler);
+
+    /**
+     * Inserts a {@link Handler} before an existing handler of this pipeline.
+     *
+     * @param baseName  the name of the existing handler.
+     * @param name      the name of the handler to insert before.
+     * @param handler   the handler to insert before.
+     *
+     * @return itself
+     */
+    Pipeline addBefore(final String baseName, final String name, final Handler handler);
+
+    /**
+     * Inserts a {@link Handler} before an existing handler of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}'s methods.
+     * @param baseName  the name of the existing handler.
+     * @param name      the name of the handler to insert before.
+     * @param handler   the handler to insert before.
+     *
+     * @return itself
+     */
+    Pipeline addBefore(final HandlerInvoker invoker, final String baseName, final String name, final Handler handler);
+
+    /**
+     * Inserts a {@link Handler} after an existing handler of this pipeline.
+     *
+     * @param baseName  the name of the existing handler.
+     * @param name      the name of the handler to insert after.
+     * @param handler   the handler to insert after.
+     *
+     * @return itself
+     */
+    Pipeline addAfter(final String baseName, final String name, final Handler handler);
+
+    /**
+     * Inserts a {@link Handler} after an existing handler of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}'s methods.
+     * @param baseName  the name of the existing handler.
+     * @param name      the name of the handler to insert after.
+     * @param handler   the handler to insert after.
+     *
+     * @return itself
+     */
+    Pipeline addAfter(final HandlerInvoker invoker, final String baseName, final String name, final Handler handler);
+
+    /**
+     * Inserts {@link Handler}s at the first position of this pipeline.
+     *
+     * @param handlers  the handlers to insert first.
+     *
+     * @return itself
+     */
+    Pipeline addFirst(final Handler... handlers);
+
+    /**
+     * Inserts {@link Handler}s at the first position of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}s's methods.
+     * @param handlers  the handlers to insert first.
+     */
+    Pipeline addFirst(final HandlerInvoker invoker, final Handler... handlers);
+
+    /**
+     * Inserts {@link Handler}s at the last position of this pipeline.
+     *
+     * @param handlers  the handlers to insert last.
+     *
+     * @return itself
+     */
+    Pipeline addLast(final Handler... handlers);
+
+    /**
+     * Inserts {@link Handler}s at the last position of this pipeline.
+     *
+     * @param invoker   the {@link HandlerInvoker} which will be used to execute the {@link Handler}s's methods.
+     * @param handlers  the handlers to insert last.
+     *
+     * @return itself
+     */
+    Pipeline addLast(final HandlerInvoker invoker, final Handler... handlers);
+
+    /**
+     * Removes the specified {@link Handler} from this pipeline.
+     *
+     * @param  handler  the {@link Handler} to remove
+     *
+     * @return the removed handler
+     */
+    Pipeline remove(final Handler handler);
+
+    /**
+     * Removes the {@link Handler} with the specified name from this pipeline.
+     *
+     * @param name  the name under which the {@link Handler} was stored.
+     *
+     * @return the removed handler
+     */
+    Handler remove(final String name);
+
+    /**
+     * Removes the {@link Handler} of the specified type from this pipeline.
+     *
+     * @param handlerType   the type of the handler.
+     * @param            the type of the handler.
+     *
+     * @return the removed handler
+     */
+     T remove(final Class handlerType);
+
+    /**
+     * Removes the first {@link Handler} in this pipeline.
+     *
+     * @return the removed handler
+     */
+    Handler removeFirst();
+
+    /**
+     * Removes the last {@link Handler} in this pipeline.
+     *
+     * @return the removed handler
+     */
+    Handler removeLast();
+
+    /**
+     * Replaces the specified {@link Handler} with a new handler in this pipeline.
+     *
+     * @param  oldHandler    the {@link Handler} to be replaced
+     * @param  newName       the name under which the replacement should be added.
+     *                       {@code null} to use the same name with the handler being replaced.
+     * @param  newHandler    the {@link Handler} which is used as replacement
+     *
+     * @return itself
+     */
+    Pipeline replace(final Handler oldHandler, final String newName, final Handler newHandler);
+
+    /**
+     * Replaces the {@link Handler} of the specified name with a new handler in this pipeline.
+     *
+     * @param oldName       the name of the {@link Handler} to be replaced.
+     * @param newName       the name under which the replacement should be added.
+     *                      {@code null} to use the same name with the handler being replaced.
+     * @param newHandler    the {@link Handler} which is used as replacement.
+     *
+     * @return the removed handler
+     */
+    Handler replace(final String oldName, final String newName, final Handler newHandler);
+
+    /**
+     * Replaces the {@link Handler} of the specified type with a new handler in this pipeline.
+     *
+     * @param  oldHandlerType   the type of the handler to be removed
+     * @param  newName          the name under which the replacement should be added.
+     *                          {@code null} to use the same name with the handler being replaced.
+     * @param  newHandler       the {@link Handler} which is used as replacement
+     *
+     * @return the removed handler
+     */
+     T replace(final Class oldHandlerType, final String newName, final Handler newHandler);
+
+    /**
+     * Returns the {@link Handler} with the specified name in this
+     * pipeline.
+     *
+     * @return the handler with the specified name.
+     *         {@code null} if there's no such handler in this pipeline.
+     */
+    Handler get(final String name);
+
+    /**
+     * Returns the {@link Handler} of the specified type in this
+     * pipeline.
+     *
+     * @return the handler of the specified handler type.
+     *         {@code null} if there's no such handler in this pipeline.
+     */
+     T get(final Class handlerType);
+
+    /**
+     * Returns the context object of the specified {@link Handler} in
+     * this pipeline.
+     *
+     * @return the context object of the specified handler.
+     *         {@code null} if there's no such handler in this pipeline.
+     */
+    HandlerContext context(final Handler handler);
+
+    /**
+     * Returns the context object of the {@link Handler} with the
+     * specified name in this pipeline.
+     *
+     * @return the context object of the handler with the specified name.
+     *         {@code null} if there's no such handler in this pipeline.
+     */
+    HandlerContext context(final String name);
+
+    /**
+     * Returns the context object of the {@link Handler} of the
+     * specified type in this pipeline.
+     *
+     * @return the context object of the handler of the specified type.
+     *         {@code null} if there's no such handler in this pipeline.
+     */
+    HandlerContext context(final Class handlerType);
+
+    /**
+     * Sends the specified {@link MessageEvent} to the first {@link InboundHandler} in this pipeline.
+     */
+    Pipeline fireInbound(final InboundMessageEvent event);
+
+    /**
+     * Sends the specified {@link MessageEvent} to the first {@link OutboundHandler} in this pipeline.
+     */
+    Pipeline fireOutbound(final OutboundMessageEvent event);
+
+    /**
+     * Received an {@link Throwable} in one of its inbound operations.
+     *
+     * This will result in having the  {@link InboundHandler#exceptionCaught(HandlerContext, MessageEvent, Throwable)}
+     * method  called of the next  {@link InboundHandler} contained in the  {@link Pipeline}.
+     */
+    Pipeline fireExceptionCaught(final MessageEvent event, final Throwable cause);
+
+    /**
+     * Invoke in the pipeline.
+     *
+     * @param event an inbound message event
+     * @param    expected return type
+     * @param    message type
+     */
+     PipelineFuture invoke(final InboundMessageEvent event);
+
+    /**
+     *
+     * @param event an inbound message event
+     * @param timeoutMillis timeout
+     * @param    expected return type
+     * @param    message type
+     */
+     PipelineFuture invoke(final InboundMessageEvent event, final long timeoutMillis);
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/PipelineException.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/PipelineException.java
new file mode 100644
index 0000000..e24f82a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/PipelineException.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+/**
+ * A {@link RuntimeException} which is thrown when a {@link Pipeline}
+ * failed to execute an operation.
+ *
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+public class PipelineException extends RuntimeException {
+
+    private static final long serialVersionUID = 3379174210419885980L;
+
+    /**
+     * Creates a new instance.
+     */
+    public PipelineException() {
+    }
+
+    /**
+     * Creates a new instance.
+     */
+    public PipelineException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    /**
+     * Creates a new instance.
+     */
+    public PipelineException(String message) {
+        super(message);
+    }
+
+    /**
+     * Creates a new instance.
+     */
+    public PipelineException(Throwable cause) {
+        super(cause);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/TypeParameterMatcher.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/TypeParameterMatcher.java
new file mode 100644
index 0000000..ff4e0c7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/TypeParameterMatcher.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
+
+import java.lang.reflect.Array;
+import java.lang.reflect.GenericArrayType;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.lang.reflect.TypeVariable;
+import java.util.IdentityHashMap;
+import java.util.Map;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ *
+ * Most of the code references the pipeline design of
+ * Netty.
+ */
+public abstract class TypeParameterMatcher {
+
+    private static final TypeParameterMatcher                                          NOOP      = new NoOpTypeParameterMatcher();
+
+    private static final ThreadLocal, TypeParameterMatcher>>  getCache  = ThreadLocal.withInitial(IdentityHashMap::new);
+
+    private static final ThreadLocal, Map>> findCache = ThreadLocal.withInitial(IdentityHashMap::new);
+
+    public static TypeParameterMatcher get(final Class parameterType) {
+        final Map, TypeParameterMatcher> getCache = TypeParameterMatcher.getCache.get();
+
+        TypeParameterMatcher matcher = getCache.get(parameterType);
+        if (matcher == null) {
+            if (parameterType == Object.class) {
+                matcher = NOOP;
+            }
+
+            if (matcher == null) {
+                matcher = new ReflectiveMatcher(parameterType);
+            }
+
+            getCache.put(parameterType, matcher);
+        }
+
+        return matcher;
+    }
+
+    public static TypeParameterMatcher find(final Object object, final Class parameterizedSuperclass,
+                                            final String typeParamName) {
+
+        final Map, Map> findCache = TypeParameterMatcher.findCache.get();
+        final Class thisClass = object.getClass();
+
+        final Map map = findCache.computeIfAbsent(thisClass, k -> Maps.newHashMap());
+
+        TypeParameterMatcher matcher = map.get(typeParamName);
+        if (matcher == null) {
+            matcher = get(find0(object, parameterizedSuperclass, typeParamName));
+            map.put(typeParamName, matcher);
+        }
+
+        return matcher;
+    }
+
+    private static Class find0(final Object object, Class parameterizedSuperclass, String typeParamName) {
+
+        final Class thisClass = object.getClass();
+        Class currentClass = thisClass;
+        for (;;) {
+            if (currentClass.getSuperclass() == parameterizedSuperclass) {
+                int typeParamIndex = -1;
+                final TypeVariable[] typeParams = currentClass.getSuperclass().getTypeParameters();
+                for (int i = 0; i < typeParams.length; i ++) {
+                    if (typeParamName.equals(typeParams[i].getName())) {
+                        typeParamIndex = i;
+                        break;
+                    }
+                }
+
+                if (typeParamIndex < 0) {
+                    throw new IllegalStateException(
+                            "unknown type parameter '" + typeParamName + "': " + parameterizedSuperclass);
+                }
+
+                final Type genericSuperType = currentClass.getGenericSuperclass();
+                if (!(genericSuperType instanceof ParameterizedType)) {
+                    return Object.class;
+                }
+
+                final Type[] actualTypeParams = ((ParameterizedType) genericSuperType).getActualTypeArguments();
+
+                Type actualTypeParam = actualTypeParams[typeParamIndex];
+                if (actualTypeParam instanceof ParameterizedType) {
+                    actualTypeParam = ((ParameterizedType) actualTypeParam).getRawType();
+                }
+                if (actualTypeParam instanceof Class) {
+                    return (Class) actualTypeParam;
+                }
+                if (actualTypeParam instanceof GenericArrayType) {
+                    Type componentType = ((GenericArrayType) actualTypeParam).getGenericComponentType();
+                    if (componentType instanceof ParameterizedType) {
+                        componentType = ((ParameterizedType) componentType).getRawType();
+                    }
+                    if (componentType instanceof Class) {
+                        return Array.newInstance((Class) componentType, 0).getClass();
+                    }
+                }
+                if (actualTypeParam instanceof TypeVariable) {
+                    // Resolved type parameter points to another type parameter.
+                    TypeVariable v = (TypeVariable) actualTypeParam;
+                    currentClass = thisClass;
+                    if (!(v.getGenericDeclaration() instanceof Class)) {
+                        return Object.class;
+                    }
+
+                    parameterizedSuperclass = (Class) v.getGenericDeclaration();
+                    typeParamName = v.getName();
+                    if (parameterizedSuperclass.isAssignableFrom(thisClass)) {
+                        continue;
+                    } else {
+                        return Object.class;
+                    }
+                }
+
+                return fail(thisClass, typeParamName);
+            }
+            currentClass = currentClass.getSuperclass();
+            if (currentClass == null) {
+                return fail(thisClass, typeParamName);
+            }
+        }
+    }
+
+    private static Class fail(final Class type, final String typeParamName) {
+        throw new IllegalStateException(
+                "cannot determine the type of the type parameter '" + typeParamName + "': " + type);
+    }
+
+    public abstract boolean match(final Object msg);
+
+    private static final class ReflectiveMatcher extends TypeParameterMatcher {
+
+        private final Class type;
+
+        ReflectiveMatcher(Class type) {
+            this.type = type;
+        }
+
+        @Override
+        public boolean match(final Object msg) {
+            return type.isInstance(msg);
+        }
+    }
+
+    private static final class NoOpTypeParameterMatcher extends TypeParameterMatcher {
+
+        @Override
+        public boolean match(final Object msg) {
+            return true;
+        }
+    }
+
+    protected TypeParameterMatcher() {}
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/InboundMessageEvent.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/InboundMessageEvent.java
new file mode 100644
index 0000000..a34045f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/InboundMessageEvent.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline.event;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * The default inbound {@link MessageEvent} implementation.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class InboundMessageEvent implements MessageEvent {
+
+    private static final AtomicLong invokeIdGenerator = new AtomicLong(0);
+
+    private final long              invokeId;
+    private final T                 message;
+
+    public InboundMessageEvent(T message) {
+        this.invokeId = invokeIdGenerator.getAndIncrement();
+        this.message = message;
+    }
+
+    @Override
+    public long getInvokeId() {
+        return invokeId;
+    }
+
+    @Override
+    public T getMessage() {
+        return message;
+    }
+
+    @Override
+    public String toString() {
+        return "InboundMessageEvent{" + "invokeId=" + invokeId + ", message=" + message + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/MessageEvent.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/MessageEvent.java
new file mode 100644
index 0000000..8741ef1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/MessageEvent.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline.event;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface MessageEvent {
+
+    long getInvokeId();
+
+    /**
+     * Returns the message.
+     */
+    T getMessage();
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/OutboundMessageEvent.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/OutboundMessageEvent.java
new file mode 100644
index 0000000..e349a70
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/event/OutboundMessageEvent.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline.event;
+
+/**
+ * The default outbound {@link MessageEvent} implementation.
+ *
+ * @author jiachun.fjc
+ */
+public abstract class OutboundMessageEvent implements MessageEvent {
+
+    private final long invokeId;
+    private final T    message;
+
+    public OutboundMessageEvent(long invokeId, T message) {
+        this.invokeId = invokeId;
+        this.message = message;
+    }
+
+    @Override
+    public long getInvokeId() {
+        return invokeId;
+    }
+
+    @Override
+    public T getMessage() {
+        return message;
+    }
+
+    @Override
+    public String toString() {
+        return "OutboundMessageEvent{" + "invokeId=" + invokeId + ", message=" + message + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/DefaultPipelineFuture.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/DefaultPipelineFuture.java
new file mode 100644
index 0000000..d7d6c4d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/DefaultPipelineFuture.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline.future;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultPipelineFuture extends CompletableFuture implements PipelineFuture {
+
+    private static final Logger                                        LOG                         = LoggerFactory
+                                                                                                       .getLogger(DefaultPipelineFuture.class);
+
+    private static final long                                          DEFAULT_TIMEOUT_NANOSECONDS = TimeUnit.SECONDS
+                                                                                                       .toNanos(30);
+
+    private static final ConcurrentMap> futures                     = Maps
+                                                                                                       .newConcurrentMapLong();
+
+    private final long                                                 invokeId;
+    private final long                                                 timeout;
+    private final long                                                 startTime                   = System.nanoTime();
+
+    public static  DefaultPipelineFuture with(final long invokeId, final long timeoutMillis) {
+        return new DefaultPipelineFuture<>(invokeId, timeoutMillis);
+    }
+
+    public static void received(final long invokeId, final Object response) {
+        final DefaultPipelineFuture future = futures.remove(invokeId);
+        if (future == null) {
+            LOG.warn("A timeout response [{}] finally returned.", response);
+            return;
+        }
+        future.doReceived(response);
+    }
+
+    private DefaultPipelineFuture(long invokeId, long timeoutMillis) {
+        this.invokeId = invokeId;
+        this.timeout = timeoutMillis > 0 ? TimeUnit.MILLISECONDS.toNanos(timeoutMillis) : DEFAULT_TIMEOUT_NANOSECONDS;
+        futures.put(invokeId, this);
+    }
+
+    @Override
+    public V getResult() throws Throwable {
+        return get(timeout, TimeUnit.NANOSECONDS);
+    }
+
+    @SuppressWarnings("unchecked")
+    private void doReceived(final Object response) {
+        if (response instanceof Throwable) {
+            completeExceptionally((Throwable) response);
+        } else {
+            complete((V) response);
+        }
+    }
+
+    // timeout scanner
+    @SuppressWarnings("all")
+    private static class TimeoutScanner implements Runnable {
+
+        public void run() {
+            for (;;) {
+                try {
+                    for (final DefaultPipelineFuture future : futures.values()) {
+                        process(future);
+                    }
+
+                    Thread.sleep(30);
+                } catch (Throwable t) {
+                    LOG.error("An exception has been caught while scanning the timeout futures.", t);
+                }
+            }
+        }
+
+        private void process(final DefaultPipelineFuture future) {
+            if (future == null || future.isDone()) {
+                return;
+            }
+
+            if (System.nanoTime() - future.startTime > future.timeout) {
+                DefaultPipelineFuture.received(future.invokeId, new TimeoutException());
+            }
+        }
+    }
+
+    static {
+        final Thread t = new Thread(new TimeoutScanner(), "timeout.scanner");
+        t.setDaemon(true);
+        t.start();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/PipelineFuture.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/PipelineFuture.java
new file mode 100644
index 0000000..3af4a55
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/future/PipelineFuture.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline.future;
+
+import java.util.concurrent.CompletionStage;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface PipelineFuture extends CompletionStage {
+
+    /**
+     * Waits for this future to be completed and get the result.
+     */
+    V getResult() throws Throwable;
+}
diff --git a/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/package-info.java b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/package-info.java
new file mode 100644
index 0000000..cb1972f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/java/com/alipay/sofa/jraft/rhea/util/pipeline/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Most of the code references the pipeline design of
+ * Netty.
+ *
+ * @author jiachun.fjc
+ */
+package com.alipay.sofa.jraft.rhea.util.pipeline;
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler b/jraft-rheakv/rheakv-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
new file mode 100644
index 0000000..a2950a0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/main/resources/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
@@ -0,0 +1,2 @@
+com.alipay.sofa.jraft.rhea.RheaKVDescribeSignalHandler
+com.alipay.sofa.jraft.rhea.RheaKVMetricsSignalHandler
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/KeyValueTool.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/KeyValueTool.java
new file mode 100644
index 0000000..850f37f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/KeyValueTool.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * @author jiachun.fjc
+ */
+public class KeyValueTool {
+
+    public static byte[] makeKey(String key) {
+        Requires.requireNonNull(key, "key");
+        return BytesUtil.writeUtf8(key);
+    }
+
+    public static byte[] makeValue(String value) {
+        Requires.requireNonNull(value, "value");
+        return BytesUtil.writeUtf8(value);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/TestUtil.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/TestUtil.java
new file mode 100644
index 0000000..7ed2697
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/TestUtil.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+
+/**
+ * @author jiachun.fjc
+ */
+public class TestUtil {
+
+    public static final int INIT_PORT = 18800;
+
+    public static List generatePeers(int n) {
+        final List peers = new ArrayList<>();
+        for (int i = 0; i < n; i++) {
+            peers.add(new PeerId("127.0.0.1", INIT_PORT + i));
+        }
+        return peers;
+    }
+
+    public static  T getByName(final Object obj, final String name, final Class type) {
+        try {
+            final Field f = obj.getClass().getDeclaredField(name);
+            f.setAccessible(true);
+            return type.cast(f.get(obj));
+        } catch (final Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/BenchmarkUtil.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/BenchmarkUtil.java
new file mode 100644
index 0000000..f950375
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/BenchmarkUtil.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark;
+
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class BenchmarkUtil {
+
+    public static final int    CONCURRENCY = 32;
+    public static final int    KEY_COUNT   = 1000000;
+
+    public static final byte[] VALUE_BYTES;
+
+    static {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] bytes = new byte[100];
+        random.nextBytes(bytes);
+        VALUE_BYTES = bytes;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/BaseRawStoreBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/BaseRawStoreBenchmark.java
new file mode 100644
index 0000000..96a98d7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/BaseRawStoreBenchmark.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.raw;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+
+public class BaseRawStoreBenchmark {
+
+    protected String          tempPath;
+    protected RocksRawKVStore kvStore;
+    protected RocksDBOptions  dbOptions;
+
+    protected void setup() throws Exception {
+        File file = getTempDir();
+        this.tempPath = file.getAbsolutePath();
+        System.out.println(this.tempPath);
+        this.kvStore = new RocksRawKVStore();
+        this.dbOptions = new RocksDBOptions();
+        this.dbOptions.setDbPath(this.tempPath);
+        this.dbOptions.setSync(false);
+        this.kvStore.init(this.dbOptions);
+    }
+
+    protected File getTempDir() throws IOException {
+        final File file = File.createTempFile("RawRocksDBTest", "test");
+        FileUtils.forceDelete(file);
+        if (file.mkdirs()) {
+            return file;
+        }
+        throw new RuntimeException("fail to make dirs");
+    }
+
+    protected void tearDown() throws Exception {
+        this.kvStore.shutdown();
+        FileUtils.forceDelete(new File(this.tempPath));
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.java
new file mode 100644
index 0000000..fda16a8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.raw;
+
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.CONCURRENCY;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class RawKVApproximateBenchmark extends BaseRawStoreBenchmark {
+
+    /**
+     //
+     // 100w keys, each value is 100 bytes.
+     //
+     // tps = 0.025 * 1000 = 25 ops/second
+     //
+     Benchmark                                                                                Mode  Cnt     Score     Error   Units
+     RawKVApproximateBenchmark.getApproximateKeysInRange                                     thrpt    3     0.025 ±   0.001  ops/ms
+     RawKVApproximateBenchmark.getApproximateKeysInRange                                      avgt    3  1304.013 ± 115.808   ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange                                    sample  693  1308.347 ±   5.718   ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.00    sample       1153.434             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.50    sample       1308.623             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.90    sample       1366.504             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.95    sample       1384.120             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.99    sample       1415.703             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.999   sample       1457.521             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p0.9999  sample       1457.521             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange:getApproximateKeysInRange·p1.00    sample       1457.521             ms/op
+     RawKVApproximateBenchmark.getApproximateKeysInRange                                        ss    3  1158.760 ± 894.444   ms/op
+     */
+
+    private static final int KEY_COUNT = 1000000;
+
+    @Setup
+    public void setup() {
+        try {
+            super.setup();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        for (int i = 0; i < KEY_COUNT; i++) {
+            byte[] key = BytesUtil.writeUtf8("approximate_" + i);
+            super.kvStore.put(key, VALUE_BYTES, null);
+        }
+    }
+
+    @TearDown
+    public void tearDown() {
+        try {
+            super.tearDown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void getApproximateKeysInRange() {
+        super.kvStore.getApproximateKeysInRange(null, BytesUtil.writeUtf8("approximate_" + (KEY_COUNT - 1)));
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(RawKVApproximateBenchmark.class.getSimpleName()) //
+            .warmupIterations(3) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(CONCURRENCY) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.java
new file mode 100644
index 0000000..8eec32e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.raw;
+
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.CONCURRENCY;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.KEY_COUNT;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class RawKVGetBenchmark extends BaseRawStoreBenchmark {
+    /**
+     //
+     // 100w keys, each value is 100 bytes.
+     //
+     // get tps = 1034.823 * 1000 = 1034823 ops/second
+     //
+     Benchmark                            Mode       Cnt     Score     Error   Units
+     RawKVGetBenchmark.get               thrpt         3  1034.823 ± 914.022  ops/ms
+     RawKVGetBenchmark.get                avgt         3     0.032 ±   0.007   ms/op
+     RawKVGetBenchmark.get              sample  14885516     0.047 ±   0.001   ms/op
+     RawKVGetBenchmark.get:get·p0.00    sample               0.003             ms/op
+     RawKVGetBenchmark.get:get·p0.50    sample               0.007             ms/op
+     RawKVGetBenchmark.get:get·p0.90    sample               0.008             ms/op
+     RawKVGetBenchmark.get:get·p0.95    sample               0.009             ms/op
+     RawKVGetBenchmark.get:get·p0.99    sample               0.016             ms/op
+     RawKVGetBenchmark.get:get·p0.999   sample               4.526             ms/op
+     RawKVGetBenchmark.get:get·p0.9999  sample              69.599             ms/op
+     RawKVGetBenchmark.get:get·p1.00    sample             248.775             ms/op
+     RawKVGetBenchmark.get                  ss         3     0.039 ±   0.100   ms/op
+     */
+
+    @Setup
+    public void setup() {
+        try {
+            super.setup();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        // insert data first
+        put();
+        long dbSize = this.kvStore.getApproximateKeysInRange(null, null);
+        System.out.println("db size = " + dbSize);
+    }
+
+    @TearDown
+    public void tearDown() {
+        try {
+            super.tearDown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void get() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] key = BytesUtil.writeUtf8("benchmark_" + random.nextInt(KEY_COUNT));
+        this.kvStore.get(key, false, null);
+    }
+
+    public void put() {
+        final List batch = Lists.newArrayListWithCapacity(100);
+        for (int i = 0; i < KEY_COUNT; i++) {
+            byte[] key = BytesUtil.writeUtf8("benchmark_" + i);
+            batch.add(new KVEntry(key, VALUE_BYTES));
+            if (batch.size() >= 100) {
+                this.kvStore.put(batch, null);
+                batch.clear();
+            }
+        }
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(RawKVGetBenchmark.class.getSimpleName()) //
+            .warmupIterations(1) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(CONCURRENCY) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.java
new file mode 100644
index 0000000..b705e1a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.raw;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.CONCURRENCY;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.KEY_COUNT;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class RawKVPutBenchmark extends BaseRawStoreBenchmark {
+
+    /**
+     //
+     // 100w keys, each value is 100 bytes.
+     //
+     // put tps = 133.658 * 1000 = 133658 ops/second
+     //
+     Benchmark                            Mode      Cnt    Score    Error   Units
+     RawKVPutBenchmark.put               thrpt        3  133.658 ± 26.637  ops/ms
+     RawKVPutBenchmark.put                avgt        3    0.233 ±  0.023   ms/op
+     RawKVPutBenchmark.put              sample  4057686    0.236 ±  0.001   ms/op
+     RawKVPutBenchmark.put:put·p0.00    sample             0.013            ms/op
+     RawKVPutBenchmark.put:put·p0.50    sample             0.219            ms/op
+     RawKVPutBenchmark.put:put·p0.90    sample             0.332            ms/op
+     RawKVPutBenchmark.put:put·p0.95    sample             0.375            ms/op
+     RawKVPutBenchmark.put:put·p0.99    sample             0.499            ms/op
+     RawKVPutBenchmark.put:put·p0.999   sample             0.818            ms/op
+     RawKVPutBenchmark.put:put·p0.9999  sample             2.726            ms/op
+     RawKVPutBenchmark.put:put·p1.00    sample            24.707            ms/op
+     RawKVPutBenchmark.put                  ss        3    0.364 ±  0.737   ms/op
+
+     */
+
+    @Setup
+    public void setup() {
+        try {
+            super.setup();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    @TearDown
+    public void tearDown() {
+        try {
+            super.tearDown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(RawKVPutBenchmark.class.getSimpleName()) //
+            .warmupIterations(1) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(CONCURRENCY) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void put() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] key = BytesUtil.writeUtf8("benchmark_" + random.nextInt(KEY_COUNT));
+        super.kvStore.put(key, VALUE_BYTES, null);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/SnapshotBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/SnapshotBenchmark.java
new file mode 100644
index 0000000..38c9459
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/raw/SnapshotBenchmark.java
@@ -0,0 +1,367 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.raw;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.Checksum;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreAccessHelper;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.ZipUtil;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.CRC64;
+
+import static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.KEY_COUNT;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+public class SnapshotBenchmark extends BaseRawStoreBenchmark {
+
+    private static final String SNAPSHOT_DIR     = "kv";
+    private static final String SNAPSHOT_ARCHIVE = "kv.zip";
+
+    public void setup() {
+        try {
+            super.setup();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        // insert data first
+        put();
+        long dbSize = this.kvStore.getApproximateKeysInRange(null, null);
+        System.out.println("db size = " + dbSize);
+    }
+
+    public void tearDown() {
+        try {
+            super.tearDown();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    public void put() {
+        final List batch = Lists.newArrayListWithCapacity(100);
+        for (int i = 0; i < KEY_COUNT * 10; i++) {
+            final byte[] key = BytesUtil.writeUtf8("benchmark_" + i);
+            batch.add(new KVEntry(key, VALUE_BYTES));
+            if (batch.size() >= 100) {
+                this.kvStore.put(batch, null);
+                batch.clear();
+            }
+        }
+    }
+
+    /**
+        -----------------------------------------------
+        db size = 10000000
+        slow save snapshot time cost: 2552
+        slow compressed file size: 41915298
+        slow compress time cost: 9173
+        slow load snapshot time cost: 5119
+        -----------------------------------------------
+        db size = 10000000
+        fast save snapshot time cost: 524
+        fast compressed file size: 41920248
+        fast compress time cost: 8807
+        fast load snapshot time cost: 3090
+        -----------------------------------------------
+        db size = 10000000
+        sst save snapshot time cost: 4296
+        sst compressed file size: 10741032
+        sst compress time cost: 2005
+        sst load snapshot time cost: 593
+        -----------------------------------------------
+        db size = 10000000
+        slow save snapshot time cost: 2248
+        slow compressed file size: 41918551
+        slow compress time cost: 8705
+        slow load snapshot time cost: 4485
+        -----------------------------------------------
+        db size = 10000000
+        fast save snapshot time cost: 508
+        fast compressed file size: 41914702
+        fast compress time cost: 8736
+        fast load snapshot time cost: 3047
+        -----------------------------------------------
+        db size = 10000000
+        sst save snapshot time cost: 4206
+        sst compressed file size: 10741032
+        sst compress time cost: 1950
+        sst load snapshot time cost: 599
+        -----------------------------------------------
+        db size = 10000000
+        slow save snapshot time cost: 2327
+        slow compressed file size: 41916640
+        slow compress time cost: 8643
+        slow load snapshot time cost: 4590
+        -----------------------------------------------
+        db size = 10000000
+        fast save snapshot time cost: 511
+        fast compressed file size: 41914533
+        fast compress time cost: 8704
+        fast load snapshot time cost: 3013
+        -----------------------------------------------
+        db size = 10000000
+        sst save snapshot time cost: 4253
+        sst compressed file size: 10741032
+        sst compress time cost: 1947
+        sst load snapshot time cost: 590
+        -----------------------------------------------
+     */
+
+    public static void main(String[] args) throws IOException {
+        for (int i = 0; i < 3; i++) {
+            SnapshotBenchmark snapshot = new SnapshotBenchmark();
+            snapshot.setup();
+            snapshot.snapshot(false, false);
+            snapshot.tearDown();
+
+            snapshot = new SnapshotBenchmark();
+            snapshot.setup();
+            snapshot.snapshot(false, true);
+            snapshot.tearDown();
+
+            snapshot = new SnapshotBenchmark();
+            snapshot.setup();
+            snapshot.snapshot(true, true);
+            snapshot.tearDown();
+        }
+    }
+
+    public void snapshot(final boolean isSstSnapshot, final boolean isFastSnapshot) throws IOException {
+        final File backupDir = new File("backup");
+        if (backupDir.exists()) {
+            FileUtils.deleteDirectory(backupDir);
+        }
+        FileUtils.forceMkdir(backupDir);
+
+        final LocalFileMeta meta = doSnapshotSave(backupDir.getAbsolutePath(), isSstSnapshot, isFastSnapshot);
+
+        this.kvStore.shutdown();
+        FileUtils.deleteDirectory(new File(this.tempPath));
+        FileUtils.forceMkdir(new File(this.tempPath));
+        this.kvStore = new RocksRawKVStore();
+        this.kvStore.init(this.dbOptions);
+
+        final String name;
+        if (isSstSnapshot) {
+            name = "sst";
+        } else {
+            if (isFastSnapshot) {
+                name = "fast";
+            } else {
+                name = "slow";
+            }
+        }
+
+        final long decompressStart = System.nanoTime();
+        final String sourceFile = Paths.get(backupDir.getAbsolutePath(), SNAPSHOT_ARCHIVE).toString();
+        ZipUtil.decompress(sourceFile, backupDir.getAbsolutePath(), new CRC64());
+        System.out.println(name + " decompress time cost: "
+                           + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - decompressStart));
+
+        final long loadStart = System.nanoTime();
+        doSnapshotLoad(backupDir.getAbsolutePath(), meta, isFastSnapshot);
+
+        System.out.println(name + " load snapshot time cost: "
+                           + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - loadStart));
+        FileUtils.deleteDirectory(backupDir);
+    }
+
+    private LocalFileMeta doSnapshotSave(final String path, final boolean isSstSnapshot, final boolean isFastSnapshot) {
+        final String snapshotPath = Paths.get(path, SNAPSHOT_DIR).toString();
+        try {
+            final long saveStart = System.nanoTime();
+            LocalFileMeta.Builder metaBuilder = null;
+            if (isSstSnapshot) {
+                doSstSnapshotSave(snapshotPath);
+            } else {
+                if (isFastSnapshot) {
+                    metaBuilder = doFastSnapshotSave(snapshotPath);
+                } else {
+                    metaBuilder = doSlowSnapshotSave(snapshotPath);
+                }
+            }
+            metaBuilder = metaBuilder == null ? LocalFileMeta.newBuilder() : metaBuilder;
+            final String name;
+            if (isSstSnapshot) {
+                name = "sst";
+            } else {
+                if (isFastSnapshot) {
+                    name = "fast";
+                } else {
+                    name = "slow";
+                }
+            }
+            System.out.println(name + " save snapshot time cost: "
+                               + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - saveStart));
+            final long compressStart = System.nanoTime();
+            doCompressSnapshot(path, metaBuilder);
+            System.out.println(name + " compressed file size: "
+                               + FileUtils.sizeOf(Paths.get(path, SNAPSHOT_ARCHIVE).toFile()));
+            System.out.println(name + " compress time cost: "
+                               + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - compressStart));
+            return metaBuilder.build();
+        } catch (final Throwable t) {
+            t.printStackTrace();
+        }
+        return null;
+    }
+
+    public boolean doSnapshotLoad(final String path, final LocalFileMeta meta, final boolean isFastSnapshot) {
+        final String snapshotPath = Paths.get(path, SNAPSHOT_DIR).toString();
+        try {
+            if (isFastSnapshot) {
+                doFastSnapshotLoad(snapshotPath);
+            } else {
+                doSlowSnapshotLoad(snapshotPath, meta);
+            }
+            return true;
+        } catch (final Throwable t) {
+            t.printStackTrace();
+            return false;
+        }
+    }
+
+    private LocalFileMeta.Builder doFastSnapshotSave(final String snapshotPath) throws Exception {
+        this.dbOptions.setFastSnapshot(true);
+        final Region region = new Region();
+        return KVStoreAccessHelper.saveSnapshot(this.kvStore, snapshotPath, region);
+    }
+
+    private LocalFileMeta.Builder doSlowSnapshotSave(final String snapshotPath) throws Exception {
+        this.dbOptions.setFastSnapshot(false);
+        final Region region = new Region();
+        return KVStoreAccessHelper.saveSnapshot(this.kvStore, snapshotPath, region);
+    }
+
+    private void doSstSnapshotSave(final String snapshotPath) throws Exception {
+        FileUtils.forceMkdir(new File(snapshotPath));
+        final List regions = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final Region r = new Region();
+            r.setId(i);
+            r.setStartKey(BytesUtil.writeUtf8("benchmark_" + i));
+            if (i < 9) {
+                r.setEndKey(BytesUtil.writeUtf8("benchmark_" + (i + 1)));
+            }
+            regions.add(r);
+        }
+        final ExecutorService executor = Executors.newFixedThreadPool(10);
+        final List> futures = Lists.newArrayList();
+        for (final Region r : regions) {
+            final Future f = executor.submit(() -> {
+                try {
+                    KVStoreAccessHelper.saveSnapshot(this.kvStore, Paths.get(snapshotPath, String.valueOf(r.getId())).toString(), r);
+                } catch (final Exception e) {
+                    e.printStackTrace();
+                }
+            });
+            futures.add(f);
+        }
+        for (final Future f : futures) {
+            try {
+                f.get();
+            } catch (final InterruptedException | ExecutionException e) {
+                e.printStackTrace();
+            }
+        }
+        executor.shutdownNow();
+    }
+
+    private void doCompressSnapshot(final String path, final LocalFileMeta.Builder metaBuilder) {
+        final String outputFile = Paths.get(path, SNAPSHOT_ARCHIVE).toString();
+        try {
+            final Checksum checksum = new CRC64();
+            ZipUtil.compress(path, SNAPSHOT_DIR, outputFile, checksum);
+            metaBuilder.setChecksum(Long.toHexString(checksum.getValue()));
+        } catch (final Throwable t) {
+            t.printStackTrace();
+        }
+    }
+
+    private void doFastSnapshotLoad(final String snapshotPath) {
+        try {
+            this.dbOptions.setFastSnapshot(true);
+            final Region region = new Region();
+            KVStoreAccessHelper.loadSnapshot(this.kvStore, snapshotPath, null, region);
+        } catch (final Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void doSlowSnapshotLoad(final String snapshotPath, final LocalFileMeta meta) {
+        try {
+            this.dbOptions.setFastSnapshot(false);
+            final Region region = new Region();
+            KVStoreAccessHelper.loadSnapshot(this.kvStore, snapshotPath, meta, region);
+        } catch (final Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void doSstSnapshotLoad(final String snapshotPath) {
+        final List regions = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final Region r = new Region();
+            r.setId(i);
+            r.setStartKey(BytesUtil.writeUtf8("benchmark_" + i));
+            if (i < 9) {
+                r.setEndKey(BytesUtil.writeUtf8("benchmark_" + (i + 1)));
+            }
+            regions.add(r);
+        }
+        final ExecutorService executor = Executors.newFixedThreadPool(10);
+        final List> futures = Lists.newArrayList();
+        for (final Region r : regions) {
+            final Future f = executor.submit(() -> {
+                try {
+                    KVStoreAccessHelper.loadSnapshot(kvStore, Paths.get(snapshotPath, String.valueOf(r.getId())).toString(), null, r);
+                } catch (final Exception e) {
+                    e.printStackTrace();
+                }
+            });
+            futures.add(f);
+        }
+        for (final Future f : futures) {
+            try {
+                f.get();
+            } catch (final InterruptedException | ExecutionException e) {
+                e.printStackTrace();
+            }
+        }
+        executor.shutdownNow();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaBenchmarkCluster.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaBenchmarkCluster.java
new file mode 100644
index 0000000..9b51c62
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaBenchmarkCluster.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.rhea;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.remoting.config.Configs;
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.errors.NotLeaderException;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.SystemPropertyUtil;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaBenchmarkCluster {
+
+    private static final String[]             CONF   = {
+            "jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_1.yaml",
+            "jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_2.yaml",
+            "jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_3.yaml" };
+
+    private volatile String                   tempDbPath;
+    private volatile String                   tempRaftPath;
+    private CopyOnWriteArrayList stores = new CopyOnWriteArrayList<>();
+
+    protected void start() throws IOException, InterruptedException {
+        SystemPropertyUtil.setProperty(Configs.NETTY_BUFFER_LOW_WATERMARK, Integer.toString(256 * 1024));
+        SystemPropertyUtil.setProperty(Configs.NETTY_BUFFER_HIGH_WATERMARK, Integer.toString(512 * 1024));
+        File file = new File("benchmark_rhea_db");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("benchmark_rhea_db");
+        if (file.mkdir()) {
+            this.tempDbPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempDbPath);
+        }
+        file = new File("benchmark_rhea_raft");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("benchmark_rhea_raft");
+        if (file.mkdir()) {
+            this.tempRaftPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempRaftPath);
+        }
+        for (String c : CONF) {
+            ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+            final RheaKVStoreOptions opts = mapper.readValue(new File(c), RheaKVStoreOptions.class);
+            RheaKVStore rheaKVStore = new DefaultRheaKVStore();
+            if (rheaKVStore.init(opts)) {
+                stores.add(rheaKVStore);
+            } else {
+                throw new RuntimeException("Fail to init rhea kv store witch conf: " + c);
+            }
+        }
+        PlacementDriverClient pdClient = stores.get(0).getPlacementDriverClient();
+        Endpoint leader1 = pdClient.getLeader(1, true, 10000);
+        System.out.println("The region 1 leader is: " + leader1);
+        //        Endpoint leader2 = pdClient.getLeader(2, true, 10000);
+        //        System.out.println("The region 2 leader is: " + leader2);
+        //        Endpoint leader3 = pdClient.getLeader(3, true, 10000);
+        //        System.out.println("The region 3 leader is: " + leader3);
+    }
+
+    protected void shutdown() throws IOException {
+        for (RheaKVStore store : stores) {
+            store.shutdown();
+        }
+        if (this.tempDbPath != null) {
+            System.out.println("removing dir: " + this.tempDbPath);
+            FileUtils.forceDelete(new File(this.tempDbPath));
+        }
+        if (this.tempRaftPath != null) {
+            System.out.println("removing dir: " + this.tempRaftPath);
+            FileUtils.forceDelete(new File(this.tempRaftPath));
+        }
+    }
+
+    public RheaKVStore getLeaderStore(long regionId) {
+        for (int i = 0; i < 20; i++) {
+            for (RheaKVStore store : stores) {
+                if (((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    return store;
+                }
+            }
+            System.out.println("fail to find leader, try again");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no leader");
+    }
+
+    public RheaKVStore getFollowerStore(long regionId) {
+        for (int i = 0; i < 20; i++) {
+            for (RheaKVStore store : stores) {
+                if (!((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    return store;
+                }
+            }
+            System.out.println("fail to find follower, try again");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no follower");
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.java
new file mode 100644
index 0000000..bf52c28
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.rhea;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.KEY_COUNT;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class RheaKVGetBenchmark extends RheaBenchmarkCluster {
+    /**
+     //
+     // 100w keys, each value is 100 bytes.
+     //
+     // get tps (mem_table命中率100%, 所以才会这么高的tps) = 925.409 * 1000 = 925409 ops/second
+     // getReadOnlySafe tps                             = 62.949 * 1000 = 62949 ops/second
+     // getReadOnlySafe tps (based raft log)            = 23.170 * 1000 = 23170 ops/second
+     //
+     Benchmark                                                     Mode       Cnt    Score     Error   Units
+     RheaKVGetBenchmark.get                                       thrpt         3  925.409 ± 538.093  ops/ms
+     RheaKVGetBenchmark.getReadOnlySafe                           thrpt         3   62.949 ±   0.762  ops/ms
+     RheaKVGetBenchmark.get                                        avgt         3    0.034 ±   0.004   ms/op
+     RheaKVGetBenchmark.getReadOnlySafe                            avgt         3    0.510 ±   0.073   ms/op
+     RheaKVGetBenchmark.get                                      sample  13892415    0.048 ±   0.001   ms/op
+     RheaKVGetBenchmark.get:get·p0.00                            sample              0.004             ms/op
+     RheaKVGetBenchmark.get:get·p0.50                            sample              0.008             ms/op
+     RheaKVGetBenchmark.get:get·p0.90                            sample              0.009             ms/op
+     RheaKVGetBenchmark.get:get·p0.95                            sample              0.010             ms/op
+     RheaKVGetBenchmark.get:get·p0.99                            sample              0.015             ms/op
+     RheaKVGetBenchmark.get:get·p0.999                           sample             20.283             ms/op
+     RheaKVGetBenchmark.get:get·p0.9999                          sample             54.460             ms/op
+     RheaKVGetBenchmark.get:get·p1.00                            sample            194.773             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe                          sample   1871951    0.513 ±   0.001   ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.00    sample              0.188             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.50    sample              0.494             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.90    sample              0.637             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.95    sample              0.689             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.99    sample              0.852             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.999   sample              1.747             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p0.9999  sample             12.976             ms/op
+     RheaKVGetBenchmark.getReadOnlySafe:getReadOnlySafe·p1.00    sample             18.383             ms/op
+     RheaKVGetBenchmark.get                                          ss         3    0.044 ±   0.055   ms/op
+     RheaKVGetBenchmark.getReadOnlySafe                              ss         3    2.668 ±  13.052   ms/op
+     */
+
+    private RheaKVStore kvStore;
+
+    @Setup
+    public void setup() {
+        try {
+            super.start();
+        } catch (IOException | InterruptedException e) {
+            e.printStackTrace();
+        }
+
+        this.kvStore = getLeaderStore(1);
+
+        // insert data first
+        put();
+    }
+
+    @TearDown
+    public void tearDown() {
+        try {
+            super.shutdown();
+        } catch (final IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void get() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] key = BytesUtil.writeUtf8("benchmark_" + random.nextInt(KEY_COUNT));
+        this.kvStore.get(key, false);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void getReadOnlySafe() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] key = BytesUtil.writeUtf8("benchmark_" + random.nextInt(KEY_COUNT));
+        this.kvStore.get(key, true);
+    }
+
+    public void put() {
+        final List batch = Lists.newArrayListWithCapacity(100);
+        for (int i = 0; i < KEY_COUNT; i++) {
+            byte[] key = BytesUtil.writeUtf8("benchmark_" + i);
+            batch.add(new KVEntry(key, VALUE_BYTES));
+            if (batch.size() >= 10) {
+                this.kvStore.bPut(batch);
+                batch.clear();
+            }
+        }
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(RheaKVGetBenchmark.class.getSimpleName()) //
+            .warmupIterations(1) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(BenchmarkUtil.CONCURRENCY) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.java
new file mode 100644
index 0000000..15bd556
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.benchmark.rhea;
+
+import java.io.IOException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.CONCURRENCY;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.KEY_COUNT;
+import static com.alipay.sofa.jraft.rhea.benchmark.BenchmarkUtil.VALUE_BYTES;
+
+/**
+ * @author jiachun.fjc
+ */
+@State(Scope.Benchmark)
+public class RheaKVPutBenchmark extends RheaBenchmarkCluster {
+
+    /**
+     //
+     // 100w keys, each value is 100 bytes.
+     //
+     // put tps = 24.548 * 1000 = 24548 ops/second
+     //
+     Benchmark                             Mode     Cnt   Score    Error   Units
+     RheaKVPutBenchmark.put               thrpt       3  24.548 ± 20.413  ops/ms
+     RheaKVPutBenchmark.put                avgt       3   1.282 ±  0.651   ms/op
+     RheaKVPutBenchmark.put              sample  750138   1.279 ±  0.005   ms/op
+     RheaKVPutBenchmark.put:put·p0.00    sample           0.403            ms/op
+     RheaKVPutBenchmark.put:put·p0.50    sample           1.163            ms/op
+     RheaKVPutBenchmark.put:put·p0.90    sample           1.798            ms/op
+     RheaKVPutBenchmark.put:put·p0.95    sample           2.032            ms/op
+     RheaKVPutBenchmark.put:put·p0.99    sample           2.712            ms/op
+     RheaKVPutBenchmark.put:put·p0.999   sample           7.717            ms/op
+     RheaKVPutBenchmark.put:put·p0.9999  sample          71.303            ms/op
+     RheaKVPutBenchmark.put:put·p1.00    sample          85.197            ms/op
+     RheaKVPutBenchmark.put                  ss       3   4.422 ±  4.274   ms/op
+
+     */
+
+    private RheaKVStore kvStore;
+
+    @Setup
+    public void setup() {
+        try {
+            super.start();
+        } catch (IOException | InterruptedException e) {
+            e.printStackTrace();
+        }
+        this.kvStore = getLeaderStore(1);
+    }
+
+    @TearDown
+    public void tearDown() {
+        try {
+            super.shutdown();
+        } catch (final IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.MILLISECONDS)
+    public void put() {
+        ThreadLocalRandom random = ThreadLocalRandom.current();
+        byte[] key = BytesUtil.writeUtf8("benchmark_" + random.nextInt(KEY_COUNT));
+        this.kvStore.bPut(key, VALUE_BYTES);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(RheaKVPutBenchmark.class.getSimpleName()) //
+            .warmupIterations(1) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .threads(CONCURRENCY) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/AbstractChaosTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/AbstractChaosTest.java
new file mode 100644
index 0000000..fc8064b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/AbstractChaosTest.java
@@ -0,0 +1,247 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.RheaKVServiceFactory;
+import com.alipay.sofa.jraft.rhea.TestUtil;
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+import com.alipay.sofa.jraft.rhea.client.RheaKVCliService;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.Constants;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.NamedThreadFactory;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class AbstractChaosTest {
+
+    private static final int    LOOP_1             = Utils.cpus();
+    private static final int    LOOP_2             = 20;
+    private static final int    INITIAL_PEER_COUNT = 5;
+    private static final int    RETRIES            = 10;
+    private static final byte[] VALUE              = BytesUtil.writeUtf8("test");
+
+    @Rule
+    public TestName             testName           = new TestName();
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+    }
+
+    @After
+    public void teardown() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+    }
+
+    @Test
+    public void chaosGetTest() throws Exception {
+        ChaosTestCluster cluster = null;
+        PeerId p1 = null;
+        PeerId p2 = null;
+        for (int l = 0; l < RETRIES; l++) {
+            final ExecutorService executor = Executors.newCachedThreadPool(new NamedThreadFactory("chaos-test", true));
+            final List> allFutures = new CopyOnWriteArrayList<>();
+            try {
+                cluster = new ChaosTestCluster(TestUtil.generatePeers(INITIAL_PEER_COUNT), getStorageType(),
+                        isAllowBatching(), isOnlyLeaderRead());
+                cluster.start();
+
+                // Before writing data, remove a node (node1) and add it back later to verify that read consistency is guaranteed.
+                p1 = cluster.getRandomPeer();
+                cluster.removePeer(p1);
+
+                final RheaKVStore store = cluster.getLeaderStore();
+                // warm up
+                store.bGet("test_key");
+
+                for (int i = 0; i < LOOP_1; i++) {
+                    final int index = i;
+                    executor.execute(() -> {
+                        for (int j = 0; j < LOOP_2; j++) {
+                            allFutures.add(store.put("test_" + index + "_" + j, VALUE));
+                        }
+                    });
+                }
+
+                // In the process of writing data, remove one node (node2)
+                p2 = cluster.getRandomPeer();
+                cluster.removePeer(p2);
+
+                // Waiting for the write to be completed
+                CompletableFuture.allOf(allFutures.toArray(new CompletableFuture[0]))
+                        .get(30, TimeUnit.SECONDS);
+                break;
+            } catch (final Exception e) {
+                System.err.println("Fail to put data, try again...");
+                e.printStackTrace();
+                new FutureGroup<>(allFutures).cancel(true);
+                if (cluster != null) {
+                    cluster.stopAll();
+                }
+                cluster = null;
+            } finally {
+                ExecutorServiceHelper.shutdownAndAwaitTermination(executor);
+            }
+        }
+        if (cluster == null) {
+            throw new RuntimeException("fail to put data, can not check data");
+        }
+        try {
+            chaosGetCheckData(cluster, p2, p1);
+        } finally {
+            cluster.stopAll();
+        }
+    }
+
+    private void chaosGetCheckData(final ChaosTestCluster cluster, final PeerId p2, final PeerId p1) {
+        // Randomly select a client to verify data consistency
+        for (int i = 0; i < LOOP_1; i++) {
+            for (int j = 0; j < LOOP_2; j++) {
+                Assert.assertArrayEquals(VALUE, cluster.getRandomStore().bGet("test_" + i + "_" + j));
+            }
+        }
+
+        // Node2 rejoins and verifies read consistency on node2
+        cluster.addPeer(p2);
+        for (int i = 0; i < LOOP_1; i++) {
+            for (int j = 0; j < LOOP_2; j++) {
+                Assert.assertArrayEquals(VALUE, cluster.getByStorePeer(p2).bGet("test_" + i + "_" + j));
+            }
+        }
+
+        // Node1 rejoins and verifies read consistency on node1 (node1 will synchronize data from leader)
+        cluster.addPeer(p1);
+        for (int i = 0; i < LOOP_1; i++) {
+            for (int j = 0; j < LOOP_2; j++) {
+                Assert.assertArrayEquals(VALUE, cluster.getByStorePeer(p1).bGet("test_" + i + "_" + j));
+            }
+        }
+    }
+
+    @Test
+    public void chaosSplittingTest() {
+        final List peerIds = TestUtil.generatePeers(INITIAL_PEER_COUNT);
+        final CliOptions opts = new CliOptions();
+        opts.setTimeoutMs(30000);
+        final RheaKVCliService cliService = RheaKVServiceFactory.createAndInitRheaKVCliService(opts);
+        final long regionId = Constants.DEFAULT_REGION_ID;
+        final long newRegionId = 2;
+        final String groupId = JRaftHelper.getJRaftGroupId(ChaosTestCluster.CLUSTER_NAME, regionId);
+        final Configuration conf = new Configuration(peerIds);
+        ChaosTestCluster cluster = null;
+        for (int l = 0; l < RETRIES; l++) {
+            final ExecutorService executor = Executors.newCachedThreadPool(new NamedThreadFactory("chaos-splitting-test", true));
+            final List> allFutures = new CopyOnWriteArrayList<>();
+            try {
+                cluster = new ChaosTestCluster(peerIds, getStorageType(),
+                        isAllowBatching(), isOnlyLeaderRead());
+                cluster.start();
+
+                final RheaKVStore store = cluster.getLeaderStore();
+                // for least keys on split
+                for (int j = 0; j < LOOP_2; j++) {
+                    store.bPut(j + "_split_", VALUE);
+                }
+
+                for (int i = 0; i < LOOP_1; i++) {
+                    final int index = i;
+                    final Future f = executor.submit(() -> {
+                        for (int j = 0; j < LOOP_2; j++) {
+                            store.bPut(index + "_split_test_" + j, VALUE);
+                        }
+                    });
+                    allFutures.add(f);
+                }
+
+                final Status st = cliService.rangeSplit(regionId, newRegionId, groupId, conf);
+                if (!st.isOk()) {
+                    System.err.println("Status:" + st);
+                    throw new RuntimeException(st.toString());
+                }
+
+                // wait for all writes finished
+                for (final Future f : allFutures) {
+                    f.get(30, TimeUnit.SECONDS);
+                }
+
+                break;
+            } catch (final Exception e) {
+                System.err.println("Fail to put data, try again...");
+                e.printStackTrace();
+                for (final Future f : allFutures) {
+                    f.cancel(true);
+                }
+                if (cluster != null) {
+                    cluster.stopAll();
+                }
+                cluster = null;
+            } finally {
+                ExecutorServiceHelper.shutdownAndAwaitTermination(executor);
+            }
+        }
+        if (cluster == null) {
+            throw new RuntimeException("fail to put data, can not check data");
+        }
+
+        try {
+            chaosSplittingCheckData(cluster);
+        } finally {
+            cluster.stopAll();
+        }
+    }
+
+    private void chaosSplittingCheckData(final ChaosTestCluster cluster) {
+        // Randomly select a client to verify data consistency
+        for (int i = 0; i < LOOP_1; i++) {
+            for (int j = 0; j < LOOP_2; j++) {
+                Assert.assertArrayEquals(VALUE, cluster.getRandomStore().bGet(i + "_split_test_" + j));
+            }
+        }
+    }
+
+    public abstract StorageType getStorageType();
+
+    public abstract boolean isAllowBatching();
+
+    public abstract boolean isOnlyLeaderRead();
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBBatchingTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBBatchingTest.java
new file mode 100644
index 0000000..4d430ac
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBBatchingTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosMemoryDBBatchingTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.Memory;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return true;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return false;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBLeaderReadBatchingTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBLeaderReadBatchingTest.java
new file mode 100644
index 0000000..03c7a38
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryDBLeaderReadBatchingTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosMemoryDBLeaderReadBatchingTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.Memory;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return true;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return true;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryLeaderReadTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryLeaderReadTest.java
new file mode 100644
index 0000000..424fa70
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryLeaderReadTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosMemoryLeaderReadTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return false;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return true;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryTest.java
new file mode 100644
index 0000000..03b4ab9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosMemoryTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosMemoryTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return false;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return false;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBBatchingTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBBatchingTest.java
new file mode 100644
index 0000000..ad64a44
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBBatchingTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosRocksDBBatchingTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return true;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return false;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadBatchingTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadBatchingTest.java
new file mode 100644
index 0000000..2f1f84f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadBatchingTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosRocksDBLeaderReadBatchingTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return true;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return true;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadTest.java
new file mode 100644
index 0000000..a636498
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBLeaderReadTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosRocksDBLeaderReadTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return false;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return true;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBTest.java
new file mode 100644
index 0000000..f4c4bf7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosRocksDBTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ChaosRocksDBTest extends AbstractChaosTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+
+    @Override
+    public boolean isAllowBatching() {
+        return false;
+    }
+
+    @Override
+    public boolean isOnlyLeaderRead() {
+        return false;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosTestCluster.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosTestCluster.java
new file mode 100644
index 0000000..fb91b93
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/chaos/ChaosTestCluster.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.chaos;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.TestUtil;
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.errors.NotLeaderException;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.options.BatchingOptions;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.options.StoreEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.configured.PlacementDriverOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RheaKVStoreOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.RocksDBOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.options.configured.StoreEngineOptionsConfigured;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.Constants;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ChaosTestCluster {
+
+    private static final Logger     LOG            = LoggerFactory.getLogger(ChaosTestCluster.class);
+
+    public static String            CLUSTER_NAME   = "chaos_test";
+    public static String            DB_PATH        = "chaos_db" + File.separator;
+    public static String            RAFT_DATA_PATH = "chaos_raft_data" + File.separator;
+
+    private final List      peerIds;
+    private final StorageType       storageType;
+    private final boolean           allowBatching;
+    private final boolean           onlyLeaderRead;
+    private final List stores         = new CopyOnWriteArrayList<>();
+
+    public ChaosTestCluster(List peerIds, StorageType storageType, boolean allowBatching, boolean onlyLeaderRead) {
+        this.peerIds = peerIds;
+        this.storageType = storageType;
+        this.allowBatching = allowBatching;
+        this.onlyLeaderRead = onlyLeaderRead;
+    }
+
+    public synchronized void start() {
+        deleteFiles();
+        final Configuration conf = new Configuration(this.peerIds);
+        final String initialServerList = conf.toString();
+        for (final PeerId p : conf.listPeers()) {
+            final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured().withFake(true) // use a fake pd
+                .config();
+            final StoreEngineOptions storeOpts = StoreEngineOptionsConfigured.newConfigured() //
+                .withStorageType(this.storageType) //
+                .withRocksDBOptions(RocksDBOptionsConfigured.newConfigured().withDbPath(DB_PATH).config()) //
+                .withRaftDataPath(RAFT_DATA_PATH) //
+                .withServerAddress(p.getEndpoint()) //
+                .withLeastKeysOnSplit(10) //
+                .config();
+            final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+                .withClusterName(CLUSTER_NAME) //
+                .withInitialServerList(initialServerList).withOnlyLeaderRead(this.onlyLeaderRead) //
+                .withStoreEngineOptions(storeOpts) //
+                .withPlacementDriverOptions(pdOpts) //
+                .withFailoverRetries(30) //
+                .withFutureTimeoutMillis(TimeUnit.SECONDS.toMillis(60)) //
+                .config();
+            BatchingOptions batchingOptions = opts.getBatchingOptions();
+            if (batchingOptions == null) {
+                batchingOptions = new BatchingOptions();
+            }
+            batchingOptions.setAllowBatching(this.allowBatching);
+            opts.setBatchingOptions(batchingOptions);
+
+            final RheaKVStore store = new DefaultRheaKVStore();
+            if (!store.init(opts)) {
+                throw new IllegalStateException("fail to init store with options: " + opts);
+            }
+            this.stores.add(store);
+        }
+        awaitLeader();
+    }
+
+    public synchronized void stopAll() {
+        for (final RheaKVStore store : this.stores) {
+            store.shutdown();
+        }
+        deleteFiles();
+    }
+
+    private void deleteFiles() {
+        final File dbFile = new File(DB_PATH);
+        if (dbFile.exists()) {
+            try {
+                FileUtils.forceDelete(dbFile);
+                LOG.info("delete db file: {}", dbFile.getAbsolutePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+
+        final File raftFile = new File(RAFT_DATA_PATH);
+        if (raftFile.exists()) {
+            try {
+                FileUtils.forceDelete(raftFile);
+                LOG.info("remove raft data: {}", raftFile.getAbsolutePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+
+    public synchronized RheaKVStore getLeaderStore() {
+        awaitLeader();
+        for (final RheaKVStore store : this.stores) {
+            if (((DefaultRheaKVStore) store).isLeader(Constants.DEFAULT_REGION_ID)) {
+                return store;
+            }
+        }
+        throw new NotLeaderException("no leader");
+    }
+
+    public synchronized RheaKVStore getByStorePeer(final PeerId peerId) {
+        awaitLeader();
+        final Endpoint endpoint = JRaftHelper.toPeer(peerId).getEndpoint();
+        for (final RheaKVStore store : this.stores) {
+            if (endpoint.equals(getSelfEndpoint(store))) {
+                return store;
+            }
+        }
+        throw new RuntimeException("fail to get peer: " + peerId);
+    }
+
+    public synchronized void removePeer(final PeerId peerId) {
+        for (int i = this.stores.size() - 1; i >= 0; i--) {
+            final RheaKVStore store = this.stores.get(i);
+            if (peerId.getEndpoint().equals(getSelfEndpoint(store))) {
+                final PlacementDriverClient pdClient = store.getPlacementDriverClient();
+                if (!pdClient.removeReplica(Constants.DEFAULT_REGION_ID, JRaftHelper.toPeer(peerId), true)) {
+                    throw new RuntimeException("fail to remove peer: " + peerId);
+                }
+                store.shutdown();
+                this.stores.remove(i);
+                this.peerIds.remove(i);
+                LOG.info("Shutdown and remove peer: {}", peerId);
+                return;
+            }
+        }
+        LOG.info("Could not find peer: {}", peerId);
+    }
+
+    public synchronized void addPeer(final PeerId peerId) {
+        if (this.peerIds.contains(peerId)) {
+            throw new RuntimeException("peerId is exist: " + peerId);
+        }
+        this.peerIds.add(peerId);
+        final Configuration conf = new Configuration(this.peerIds);
+        final String initialServerList = conf.toString();
+        final PlacementDriverOptions pdOpts = PlacementDriverOptionsConfigured.newConfigured().withFake(true) // use a fake pd
+            .config();
+        final StoreEngineOptions storeOpts = StoreEngineOptionsConfigured.newConfigured() //
+            .withStorageType(this.storageType) //
+            .withRocksDBOptions(RocksDBOptionsConfigured.newConfigured().withDbPath(DB_PATH).config()) //
+            .withRaftDataPath(RAFT_DATA_PATH) //
+            .withServerAddress(peerId.getEndpoint()) //
+            .config();
+        final RheaKVStoreOptions opts = RheaKVStoreOptionsConfigured.newConfigured() //
+            .withClusterName("chaos_test") //
+            .withInitialServerList(initialServerList).withStoreEngineOptions(storeOpts) //
+            .withPlacementDriverOptions(pdOpts) //
+            .config();
+        BatchingOptions batchingOptions = opts.getBatchingOptions();
+        if (batchingOptions == null) {
+            batchingOptions = new BatchingOptions();
+        }
+        batchingOptions.setAllowBatching(this.allowBatching);
+        opts.setBatchingOptions(batchingOptions);
+
+        final RheaKVStore store = new DefaultRheaKVStore();
+        if (!store.init(opts)) {
+            throw new IllegalStateException("fail to init store with options: " + opts);
+        }
+
+        final RheaKVStore leader = getLeaderStore();
+        final PlacementDriverClient pdClient = leader.getPlacementDriverClient();
+        if (!pdClient.addReplica(Constants.DEFAULT_REGION_ID, JRaftHelper.toPeer(peerId), true)) {
+            throw new RuntimeException("fail to add peer: " + peerId);
+        }
+        this.stores.add(store);
+        awaitLeader();
+    }
+
+    public synchronized RheaKVStore getRandomStore() {
+        final ThreadLocalRandom random = ThreadLocalRandom.current();
+        return this.stores.get(random.nextInt(this.stores.size()));
+    }
+
+    public synchronized PeerId getRandomPeer() {
+        final ThreadLocalRandom random = ThreadLocalRandom.current();
+        return this.peerIds.get(random.nextInt(this.peerIds.size()));
+    }
+
+    public synchronized void randomTransferLeader() {
+        final RheaKVStore leader = getLeaderStore();
+        final Endpoint leaderEndpoint = getSelfEndpoint(leader);
+        final PlacementDriverClient pdClient = leader.getPlacementDriverClient();
+        final Peer randomPeer = JRaftHelper.toPeer(getRandomPeer());
+        boolean result = pdClient.transferLeader(Constants.DEFAULT_REGION_ID, randomPeer, false);
+        if (!result) {
+            throw new RuntimeException("fail to transfer leader [" + leaderEndpoint + " --> " + randomPeer);
+        }
+        LOG.info("Transfer leader from {} to {}", leaderEndpoint, randomPeer.getEndpoint());
+    }
+
+    public synchronized Endpoint getSelfEndpoint(final RheaKVStore store) {
+        final StoreEngine storeEngine = TestUtil.getByName(store, "storeEngine", StoreEngine.class);
+        return storeEngine.getSelfEndpoint();
+    }
+
+    public synchronized void awaitLeader() {
+        for (int i = 0; i < 100; i++) {
+            for (final RheaKVStore store : this.stores) {
+                if (((DefaultRheaKVStore) store).isLeader(Constants.DEFAULT_REGION_ID)) {
+                    return;
+                }
+            }
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("wait leader timeout");
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/ListFailoverFutureTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/ListFailoverFutureTest.java
new file mode 100644
index 0000000..8b3e84b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/ListFailoverFutureTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.failover.ListRetryCallable;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryRunner;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.FailoverClosureImpl;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.ListFailoverFuture;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ListFailoverFutureTest {
+
+    private final ConcurrentMap counters = new ConcurrentHashMap<>();
+
+    @Test
+    public void setFailureTest() throws ExecutionException, InterruptedException {
+        final int reties = 5;
+        final int start = 1;
+        final int end = 20;
+        final FutureGroup> futureGroup = scan(start, end, reties, null);
+        final List resultList = Lists.newArrayList();
+        for (final CompletableFuture> future : futureGroup.futures()) {
+            resultList.addAll(FutureHelper.get(future));
+        }
+        final List expectedList = new ArrayList<>();
+        for (int i = start; i < end; i++) {
+            expectedList.add(i);
+        }
+        Assert.assertArrayEquals(expectedList.toArray(), resultList.toArray());
+        System.out.println("all result=" + resultList);
+    }
+
+    private FutureGroup> scan(final int start, final int end, final int retriesLeft,
+                                            final Throwable lastCause) {
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        final List>> futureList = Lists.newArrayList();
+        final int mid = ((end - start) / 2) + start;
+        for (int i = 0; i < 2; i++) {
+            final int subStart = i == 0 ? start : mid;
+            final int subEnd = i == 0 ? mid : end;
+            if (subEnd - subStart > 0) {
+                final ListRetryCallable retryCallable = retryCause -> scan(subStart, subEnd,
+                        retriesLeft - 1, retryCause);
+                final ListFailoverFuture future = new ListFailoverFuture<>(retriesLeft, retryCallable);
+                regionScan(subStart, subEnd, future, retriesLeft, lastError);
+                futureList.add(future);
+            }
+        }
+        return new FutureGroup<>(futureList);
+    }
+
+    @SuppressWarnings("unused")
+    private void regionScan(final int start, final int end, final CompletableFuture> future,
+                            final int retriesLeft, final Errors lastCause) {
+        System.out.println("start=" + start + ", end=" + end);
+        final RetryRunner retryRunner = retryCause -> regionScan(start, end, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure> closure = new FailoverClosureImpl<>(future, false,
+                retriesLeft, retryRunner);
+        if (getCounter(start, end).incrementAndGet() < 2) {
+            System.err.println("fail: " + start + " - " + end);
+            closure.failure(Errors.INVALID_REGION_MEMBERSHIP);
+        } else {
+            final List result = new ArrayList<>();
+            for (int i = start; i < end; i++) {
+                result.add(i);
+            }
+            closure.success(result);
+        }
+    }
+
+    private AtomicInteger getCounter(final int start, final int end) {
+        final String key = start + "_" + end;
+        AtomicInteger counter = this.counters.get(key);
+        if (counter == null) {
+            AtomicInteger newCounter = new AtomicInteger();
+            counter = this.counters.putIfAbsent(key, newCounter);
+            if (counter == null) {
+                counter = newCounter;
+            }
+        }
+        return counter;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/MapFailoverFutureTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/MapFailoverFutureTest.java
new file mode 100644
index 0000000..1f5a89a
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/MapFailoverFutureTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.client.failover.FailoverClosure;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryCallable;
+import com.alipay.sofa.jraft.rhea.client.failover.RetryRunner;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.FailoverClosureImpl;
+import com.alipay.sofa.jraft.rhea.client.failover.impl.MapFailoverFuture;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+
+/**
+ * @author jiachun.fjc
+ */
+public class MapFailoverFutureTest {
+
+    private final ConcurrentMap counters = new ConcurrentHashMap<>();
+
+    @Test
+    public void setFailureTest() throws ExecutionException, InterruptedException {
+        final int reties = 5;
+        final int start = 1;
+        final int end = 20;
+        final FutureGroup> futureGroup = multiGet(start, end, reties, null);
+        final Map resultList = Maps.newHashMap();
+        for (final CompletableFuture> future : futureGroup.futures()) {
+            resultList.putAll(FutureHelper.get(future));
+        }
+        final List expectedList = new ArrayList<>();
+        for (int i = start; i < end; i++) {
+            expectedList.add(i);
+        }
+        Assert.assertArrayEquals(expectedList.toArray(), resultList.keySet().toArray());
+        System.out.println("all result=" + resultList);
+    }
+
+    private FutureGroup> multiGet(final int start, final int end, final int retriesLeft,
+                                                        final Throwable lastCause) {
+        final Errors lastError = lastCause == null ? null : Errors.forException(lastCause);
+        final List>> futureList = Lists.newArrayList();
+        final int mid = ((end - start) / 2) + start;
+        for (int i = 0; i < 2; i++) {
+            final int subStart = i == 0 ? start : mid;
+            final int subEnd = i == 0 ? mid : end;
+            if (subEnd - subStart > 0) {
+                final RetryCallable> retryCallable = retryCause -> multiGet(subStart, subEnd,
+                        retriesLeft - 1, retryCause);
+                final MapFailoverFuture future = new MapFailoverFuture<>(retriesLeft, retryCallable);
+                regionMultiGet(subStart, subEnd, future, retriesLeft, lastError);
+                futureList.add(future);
+            }
+        }
+        return new FutureGroup<>(futureList);
+    }
+
+    @SuppressWarnings("unused")
+    private void regionMultiGet(final int start, final int end, final CompletableFuture> future,
+                                final int retriesLeft, final Errors lastCause) {
+        System.out.println("start=" + start + ", end=" + end);
+        final RetryRunner retryRunner = retryCause -> regionMultiGet(start, end, future, retriesLeft - 1,
+                retryCause);
+        final FailoverClosure> closure = new FailoverClosureImpl<>(future, false,
+                retriesLeft, retryRunner);
+        if (getCounter(start, end).incrementAndGet() < 2) {
+            System.err.println("fail: " + start + " - " + end);
+            closure.failure(Errors.INVALID_REGION_MEMBERSHIP);
+        } else {
+            final Map result = Maps.newHashMap();
+            for (int i = start; i < end; i++) {
+                result.put(i, i);
+            }
+            closure.success(result);
+        }
+    }
+
+    private AtomicInteger getCounter(final int start, final int end) {
+        final String key = start + "_" + end;
+        AtomicInteger counter = this.counters.get(key);
+        if (counter == null) {
+            AtomicInteger newCounter = new AtomicInteger();
+            counter = this.counters.putIfAbsent(key, newCounter);
+            if (counter == null) {
+                counter = newCounter;
+            }
+        }
+        return counter;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTableTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTableTest.java
new file mode 100644
index 0000000..e6d5fe4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RegionRouteTableTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.KeyValueTool;
+import com.alipay.sofa.jraft.rhea.errors.RouteTableException;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RegionRouteTableTest {
+
+    @Test
+    public void splitRegionTest() {
+        RegionRouteTable table = new RegionRouteTable();
+        Region region = makeRegion(-1, null, null);
+        table.addOrUpdateRegion(region);
+        Region newRegion = makeRegion(1, BytesUtil.writeUtf8("t"), null);
+        table.splitRegion(-1, newRegion);
+        Region found = table.findRegionByKey(BytesUtil.writeUtf8("a"));
+        assertEquals(-1, found.getId());
+        found = table.findRegionByKey(BytesUtil.writeUtf8("w"));
+        assertEquals(1, found.getId());
+    }
+
+    @Test
+    public void findRegionByKeyTest() {
+        // case-1
+        {
+            RegionRouteTable table = new RegionRouteTable();
+            Region region = makeRegion(-1, null, null);
+            table.addOrUpdateRegion(region);
+            Region found = table.findRegionByKey(new byte[0]);
+            assertEquals(region, found);
+        }
+        // case-2
+        {
+            RegionRouteTable table = new RegionRouteTable();
+            Region r1 = makeRegion(1, null, KeyValueTool.makeKey("c"));
+            Region r2 = makeRegion(2, KeyValueTool.makeKey("c"), KeyValueTool.makeKey("e"));
+            Region r3 = makeRegion(3, KeyValueTool.makeKey("e"), null);
+            table.addOrUpdateRegion(r1);
+            table.addOrUpdateRegion(r2);
+            table.addOrUpdateRegion(r3);
+            Region found = table.findRegionByKey(KeyValueTool.makeKey("abs"));
+            assertEquals(r1.getId(), found.getId());
+            found = table.findRegionByKey(KeyValueTool.makeKey("cde"));
+            assertEquals(r2.getId(), found.getId());
+            found = table.findRegionByKey(KeyValueTool.makeKey("efg"));
+            assertEquals(r3.getId(), found.getId());
+        }
+    }
+
+    @Test
+    public void findRegionsByKeyRangeTest() {
+        // case-1
+        {
+            RegionRouteTable table = new RegionRouteTable();
+            Region region = makeRegion(-1, null, null);
+            table.addOrUpdateRegion(region);
+            List regionList = table.findRegionsByKeyRange(KeyValueTool.makeKey("a"), KeyValueTool.makeKey("w"));
+            assertEquals(1, regionList.size());
+        }
+        // case-2
+        {
+            RegionRouteTable table = new RegionRouteTable();
+            Region r1 = makeRegion(1, null, KeyValueTool.makeKey("c"));
+            Region r2 = makeRegion(2, KeyValueTool.makeKey("c"), KeyValueTool.makeKey("e"));
+            Region r3 = makeRegion(3, KeyValueTool.makeKey("e"), null);
+            table.addOrUpdateRegion(r1);
+            table.addOrUpdateRegion(r2);
+            table.addOrUpdateRegion(r3);
+            List foundList = table.findRegionsByKeyRange(KeyValueTool.makeKey("adc"),
+                KeyValueTool.makeKey("def"));
+            assertEquals(2, foundList.size());
+            assertEquals(r1.getId(), foundList.get(0).getId());
+            assertEquals(r2.getId(), foundList.get(1).getId());
+
+            foundList = table.findRegionsByKeyRange(KeyValueTool.makeKey("c"), KeyValueTool.makeKey("def"));
+            assertEquals(1, foundList.size());
+            assertEquals(r2.getId(), foundList.get(0).getId());
+        }
+        // case-3
+        {
+            RegionRouteTable table = new RegionRouteTable();
+            Region r1 = makeRegion(1, null, KeyValueTool.makeKey("c"));
+            Region r2 = makeRegion(2, KeyValueTool.makeKey("c"), KeyValueTool.makeKey("e"));
+            Region r3 = makeRegion(3, KeyValueTool.makeKey("e"), KeyValueTool.makeKey("g"));
+            Region r4 = makeRegion(4, KeyValueTool.makeKey("g"), KeyValueTool.makeKey("i"));
+            Region r5 = makeRegion(5, KeyValueTool.makeKey("i"), KeyValueTool.makeKey("k"));
+            Region r6 = makeRegion(6, KeyValueTool.makeKey("k"), KeyValueTool.makeKey("n"));
+            Region r7 = makeRegion(7, KeyValueTool.makeKey("n"), null);
+            table.addOrUpdateRegion(r1);
+            table.addOrUpdateRegion(r2);
+            table.addOrUpdateRegion(r3);
+            table.addOrUpdateRegion(r4);
+            table.addOrUpdateRegion(r5);
+            table.addOrUpdateRegion(r6);
+            table.addOrUpdateRegion(r7);
+            List foundList = table.findRegionsByKeyRange(KeyValueTool.makeKey("adc"),
+                KeyValueTool.makeKey("def"));
+            assertEquals(2, foundList.size());
+            assertEquals(r1.getId(), foundList.get(0).getId());
+            assertEquals(r2.getId(), foundList.get(1).getId());
+
+            foundList = table.findRegionsByKeyRange(KeyValueTool.makeKey("c"), KeyValueTool.makeKey("kf"));
+            assertEquals(5, foundList.size());
+            assertEquals(r2.getId(), foundList.get(0).getId());
+            assertEquals(r3.getId(), foundList.get(1).getId());
+            assertEquals(r4.getId(), foundList.get(2).getId());
+            assertEquals(r5.getId(), foundList.get(3).getId());
+            assertEquals(r6.getId(), foundList.get(4).getId());
+
+        }
+    }
+
+    @Test(expected = RouteTableException.class)
+    public void brokenTest() {
+        RegionRouteTable table = new RegionRouteTable();
+        // Region r1 = makeRegion(1, null, KeyValueTool.makeKey("c"));
+        Region r2 = makeRegion(2, KeyValueTool.makeKey("c"), KeyValueTool.makeKey("e"));
+        Region r3 = makeRegion(3, KeyValueTool.makeKey("e"), KeyValueTool.makeKey("g"));
+        Region r4 = makeRegion(4, KeyValueTool.makeKey("g"), KeyValueTool.makeKey("i"));
+        Region r5 = makeRegion(5, KeyValueTool.makeKey("i"), KeyValueTool.makeKey("k"));
+        Region r6 = makeRegion(6, KeyValueTool.makeKey("k"), KeyValueTool.makeKey("n"));
+        Region r7 = makeRegion(7, KeyValueTool.makeKey("n"), null);
+        table.addOrUpdateRegion(r2);
+        table.addOrUpdateRegion(r3);
+        table.addOrUpdateRegion(r4);
+        table.addOrUpdateRegion(r5);
+        table.addOrUpdateRegion(r6);
+        table.addOrUpdateRegion(r7);
+        table.findRegionByKey(KeyValueTool.makeKey("a"));
+    }
+
+    Region makeRegion(long id, byte[] startKey, byte[] endKey) {
+        Region region = new Region();
+        region.setId(id);
+        region.setStartKey(startKey);
+        region.setEndKey(endKey);
+        region.setRegionEpoch(new RegionEpoch(-1, -1));
+        return region;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancerTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancerTest.java
new file mode 100644
index 0000000..0c1a6f1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/client/RoundRobinLoadBalancerTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.client;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RoundRobinLoadBalancerTest {
+
+    @Test
+    public void selectTest() {
+        List elements = Lists.newArrayList();
+        elements.add(0);
+        elements.add(1);
+        elements.add(2);
+        elements.add(3);
+        elements.add(4);
+        RoundRobinLoadBalancer balancer1 = RoundRobinLoadBalancer.getInstance(1);
+        Assert.assertEquals(0, balancer1.select(elements).intValue());
+        Assert.assertEquals(1, balancer1.select(elements).intValue());
+
+        RoundRobinLoadBalancer balancer2 = RoundRobinLoadBalancer.getInstance(2);
+        Assert.assertEquals(0, balancer2.select(elements).intValue());
+        Assert.assertEquals(1, balancer2.select(elements).intValue());
+        Assert.assertEquals(2, balancer2.select(elements).intValue());
+        Assert.assertEquals(3, balancer2.select(elements).intValue());
+        Assert.assertEquals(4, balancer2.select(elements).intValue());
+        Assert.assertEquals(0, balancer2.select(elements).intValue());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaHeartbeatTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaHeartbeatTest.java
new file mode 100644
index 0000000..62b2f98
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaHeartbeatTest.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pd;
+
+import java.io.IOException;
+import java.util.UUID;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.junit.Assert;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaHeartbeatTest extends RheaKVTestCluster {
+
+    private static final NamedThreadFactory threadFactory = new NamedThreadFactory("heartbeat_test", true);
+
+    public static void main(String[] args) throws Exception {
+        final RheaHeartbeatTest heartbeatTest = new RheaHeartbeatTest();
+        heartbeatTest.start();
+
+        final Thread thread = threadFactory.newThread(() -> {
+            for (;;) {
+                heartbeatTest.putAndGetValue();
+                try {
+                    Thread.sleep(500);
+                } catch (InterruptedException e) {
+                    break;
+                }
+            }
+        });
+        thread.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            try {
+                heartbeatTest.shutdown();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }));
+    }
+
+    private void putAndGetValue() {
+        final RheaKVStore store = getLeaderStore(ThreadLocalRandom.current().nextInt(1, 2));
+        final String key = UUID.randomUUID().toString();
+        final byte[] value = makeValue(UUID.randomUUID().toString());
+        store.bPut(key, value);
+        final byte[] newValue = store.bGet(key);
+        Assert.assertArrayEquals(value, newValue);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaKVTestCluster.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaKVTestCluster.java
new file mode 100644
index 0000000..b7a9fae
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/pd/RheaKVTestCluster.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pd;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.errors.NotLeaderException;
+import com.alipay.sofa.jraft.rhea.options.RegionEngineOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVTestCluster {
+
+    private static final String[]             CONF   = { "/pd_conf/rhea_pd_test_1.yaml", //
+            "/pd_conf/rhea_pd_test_2.yaml", //
+            "/pd_conf/rhea_pd_test_3.yaml" //
+                                                     };
+
+    private volatile String                   tempDbPath;
+    private volatile String                   tempRaftPath;
+    private CopyOnWriteArrayList stores = new CopyOnWriteArrayList<>();
+
+    protected void start() throws IOException, InterruptedException {
+        System.out.println("RheaKVTestCluster init ...");
+        File file = new File("rhea_pd_db");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("rhea_pd_db");
+        if (file.mkdir()) {
+            this.tempDbPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempDbPath);
+        }
+        file = new File("rhea_pd_raft");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("rhea_pd_raft");
+        if (file.mkdir()) {
+            this.tempRaftPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempRaftPath);
+        }
+
+        final Set regionIds = new HashSet<>();
+        for (final String c : CONF) {
+            final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+            final InputStream in = RheaKVTestCluster.class.getResourceAsStream(c);
+            final RheaKVStoreOptions opts = mapper.readValue(in, RheaKVStoreOptions.class);
+            for (final RegionEngineOptions rOpts : opts.getStoreEngineOptions().getRegionEngineOptionsList()) {
+                regionIds.add(rOpts.getRegionId());
+            }
+            final RheaKVStore rheaKVStore = new DefaultRheaKVStore();
+            if (rheaKVStore.init(opts)) {
+                stores.add(rheaKVStore);
+            } else {
+                System.err.println("Fail to init rhea kv store witch conf: " + c);
+            }
+        }
+        final PlacementDriverClient pdClient = stores.get(0).getPlacementDriverClient();
+        for (final Long regionId : regionIds) {
+            final Endpoint leader = pdClient.getLeader(regionId, true, 10000);
+            System.out.println("The region " + regionId + " leader is: " + leader);
+        }
+    }
+
+    protected void shutdown() throws IOException {
+        System.out.println("RheaKVTestCluster shutdown ...");
+        for (RheaKVStore store : stores) {
+            store.shutdown();
+        }
+        if (this.tempDbPath != null) {
+            System.out.println("removing dir: " + this.tempDbPath);
+            FileUtils.forceDelete(new File(this.tempDbPath));
+        }
+        if (this.tempRaftPath != null) {
+            System.out.println("removing dir: " + this.tempRaftPath);
+            FileUtils.forceDelete(new File(this.tempRaftPath));
+        }
+        System.out.println("RheaKVTestCluster shutdown complete");
+    }
+
+    protected RheaKVStore getLeaderStore(long regionId) {
+        for (int i = 0; i < 20; i++) {
+            for (RheaKVStore store : stores) {
+                if (((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    return store;
+                }
+            }
+            System.out.println("fail to find leader, try again");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no leader");
+    }
+
+    protected RheaKVStore getFollowerStore(long regionId) {
+        for (int i = 0; i < 20; i++) {
+            for (RheaKVStore store : stores) {
+                if (!((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    return store;
+                }
+            }
+            System.out.println("fail to find follower, try again");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no follower");
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/serialization/SerializerTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/serialization/SerializerTest.java
new file mode 100644
index 0000000..2f8bfad
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/serialization/SerializerTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.serialization;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.storage.KVOperation;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jiachun.fjc
+ */
+public class SerializerTest {
+
+    @Test
+    public void readObjectTest() {
+        final Serializer serializer = Serializers.getDefault();
+        final KVOperation op = KVOperation.createPut(BytesUtil.writeUtf8("key"), BytesUtil.writeUtf8("value"));
+        final byte[] bytes = serializer.writeObject(op);
+        final ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.length);
+        buffer.put(bytes);
+        buffer.flip();
+
+        final KVOperation op1 = serializer.readObject(bytes, KVOperation.class);
+        final KVOperation op2 = serializer.readObject(buffer, KVOperation.class);
+
+        assertArrayEquals(op1.getKey(), op.getKey());
+        assertArrayEquals(op1.getValue(), op.getValue());
+        assertEquals(op.getOp(), op.getOp());
+
+        assertArrayEquals(op1.getKey(), op2.getKey());
+        assertArrayEquals(op1.getValue(), op2.getValue());
+        assertEquals(op.getOp(), op2.getOp());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStateMachineTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStateMachineTest.java
new file mode 100644
index 0000000..9490301
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStateMachineTest.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.rhea.StateListenerContainer;
+import com.alipay.sofa.jraft.rhea.StoreEngine;
+import com.alipay.sofa.jraft.rhea.client.pd.FakePlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.util.internal.ThrowUtil;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+import static com.alipay.sofa.jraft.core.State.STATE_ERROR;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author jiachun.fjc
+ */
+public class KVStateMachineTest {
+
+    private static final int APPLY_COUNT   = 100;
+    private static final int SUCCESS_COUNT = 10;
+
+    private RaftGroupService raftGroupService;
+    private RaftRawKVStore   raftRawKVStore;
+    private File             raftDataPath;
+
+    @Before
+    public void setup() throws IOException, InterruptedException {
+        final Region region = new Region();
+        region.setId(1);
+        final StoreEngine storeEngine = new MockStoreEngine();
+        final KVStoreStateMachine fsm = new KVStoreStateMachine(region, storeEngine);
+        final NodeOptions nodeOpts = new NodeOptions();
+        final Configuration conf = new Configuration();
+        conf.addPeer(PeerId.parsePeer("127.0.0.1:8081"));
+        nodeOpts.setInitialConf(conf);
+        nodeOpts.setFsm(fsm);
+
+        final String raftDataPath = "raft_st_test";
+        this.raftDataPath = new File(raftDataPath);
+        if (this.raftDataPath.exists()) {
+            FileUtils.forceDelete(this.raftDataPath);
+        }
+        FileUtils.forceMkdir(this.raftDataPath);
+
+        final Path logUri = Paths.get(raftDataPath, "log");
+        nodeOpts.setLogUri(logUri.toString());
+
+        final Path meteUri = Paths.get(raftDataPath, "meta");
+        nodeOpts.setRaftMetaUri(meteUri.toString());
+
+        final Path snapshotUri = Paths.get(raftDataPath, "snapshot");
+        nodeOpts.setSnapshotUri(snapshotUri.toString());
+
+        final Endpoint serverAddress = new Endpoint("127.0.0.1", 8081);
+        final PeerId serverId = new PeerId(serverAddress, 0);
+        this.raftGroupService = new RaftGroupService("st_test", serverId, nodeOpts, null, true);
+
+        final Node node = this.raftGroupService.start(false);
+
+        for (int i = 0; i < 100; i++) {
+            if (node.isLeader()) {
+                break;
+            }
+            Thread.sleep(100);
+        }
+
+        final RawKVStore rawKVStore = storeEngine.getRawKVStore();
+        this.raftRawKVStore = new RaftRawKVStore(node, rawKVStore, null);
+    }
+
+    @After
+    public void tearDown() throws IOException {
+        if (this.raftGroupService != null) {
+            this.raftGroupService.shutdown();
+            try {
+                this.raftGroupService.join();
+            } catch (final InterruptedException e) {
+                ThrowUtil.throwException(e);
+            }
+        }
+        if (this.raftDataPath.exists()) {
+            FileUtils.forceDelete(this.raftDataPath);
+        }
+    }
+
+    @Test
+    public void failApplyTest() throws Exception {
+        final CountDownLatch latch = new CountDownLatch(APPLY_COUNT);
+        final List closures = new ArrayList<>();
+        final BlockingQueue successQueue = new ArrayBlockingQueue<>(APPLY_COUNT);
+        final BlockingQueue failQueue = new ArrayBlockingQueue<>(APPLY_COUNT);
+        assertTrue(this.raftGroupService.getRaftNode().isLeader());
+        for (int i = 0; i < SUCCESS_COUNT; i++) {
+            final KVStoreClosure c = new BaseKVStoreClosure() {
+
+                @Override
+                public void run(Status status) {
+                    successQueue.add(status);
+                    latch.countDown();
+                }
+            };
+            closures.add(c);
+        }
+        for (int i = SUCCESS_COUNT; i < APPLY_COUNT; i++) {
+            final KVStoreClosure c = new BaseKVStoreClosure() {
+
+                @Override
+                public void run(Status status) {
+                    failQueue.add(status);
+                    latch.countDown();
+                }
+            };
+            closures.add(c);
+        }
+
+        for (int i = 0; i < SUCCESS_COUNT; i++) {
+            final byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            this.raftRawKVStore.put(bytes, bytes, closures.get(i));
+        }
+
+        for (int i = SUCCESS_COUNT; i < APPLY_COUNT; i++) {
+            final byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            this.raftRawKVStore.merge(bytes, bytes, closures.get(i));
+        }
+
+        latch.await();
+
+        final Node node = this.raftGroupService.getRaftNode();
+        assertFalse(node.isLeader());
+        final Field field = node.getClass().getDeclaredField("state");
+        field.setAccessible(true);
+        assertEquals(field.get(node), STATE_ERROR);
+        assertEquals(SUCCESS_COUNT, successQueue.size());
+        assertEquals(APPLY_COUNT - SUCCESS_COUNT, failQueue.size());
+        while (true) {
+            final Status st = successQueue.poll();
+            if (st == null) {
+                break;
+            }
+            assertTrue(st.isOk());
+        }
+
+        while (true) {
+            final Status st = failQueue.poll();
+            if (st == null) {
+                break;
+            }
+            assertFalse(st.isOk());
+            assertTrue(st.getRaftError() == RaftError.ESTATEMACHINE || st.getRaftError() == RaftError.EPERM);
+        }
+    }
+
+    static class MockKVStore extends MemoryRawKVStore {
+
+        private int putIndex = 0;
+
+        @Override
+        public void put(byte[] key, byte[] value, KVStoreClosure closure) {
+            if (this.putIndex++ < SUCCESS_COUNT) {
+                if (closure != null) {
+                    closure.setData(value);
+                    closure.run(Status.OK());
+                }
+            } else {
+                throw new RuntimeException("fail put test");
+            }
+        }
+
+        @Override
+        public void merge(byte[] key, byte[] value, KVStoreClosure closure) {
+            if (this.putIndex++ < SUCCESS_COUNT) {
+                if (closure != null) {
+                    closure.setData(value);
+                    closure.run(Status.OK());
+                }
+            } else {
+                throw new RuntimeException("fail merge test");
+            }
+        }
+
+        @Override
+        void doSnapshotSave(MemoryKVStoreSnapshotFile snapshotFile, String snapshotPath, Region region)
+                                                                                                       throws Exception {
+            super.doSnapshotSave(snapshotFile, snapshotPath, region);
+            snapshotFile.writeToFile(snapshotPath, "putIndex", new PutIndex(this.putIndex));
+        }
+
+        @Override
+        void doSnapshotLoad(MemoryKVStoreSnapshotFile snapshotFile, String snapshotPath) throws Exception {
+            super.doSnapshotLoad(snapshotFile, snapshotPath);
+            final PutIndex p = snapshotFile.readFromFile(snapshotPath, "putIndex", PutIndex.class);
+            this.putIndex = p.data();
+        }
+
+        class PutIndex extends MemoryKVStoreSnapshotFile.Persistence {
+
+            public PutIndex(Integer data) {
+                super(data);
+            }
+        }
+    }
+
+    static class MockStoreEngine extends StoreEngine {
+
+        private final MockKVStore     mockKVStore        = new MockKVStore();
+        private final ExecutorService leaderStateTrigger = Executors.newSingleThreadExecutor();
+
+        public MockStoreEngine() {
+            super(new MockPlacementDriverClient(), new StateListenerContainer<>());
+        }
+
+        @Override
+        public BatchRawKVStore getRawKVStore() {
+            return this.mockKVStore;
+        }
+
+        @Override
+        public ExecutorService getRaftStateTrigger() {
+            return this.leaderStateTrigger;
+        }
+    }
+
+    static class MockPlacementDriverClient extends FakePlacementDriverClient {
+
+        public MockPlacementDriverClient() {
+            super(1, "st_test");
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStoreAccessHelper.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStoreAccessHelper.java
new file mode 100644
index 0000000..18bdb24
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/KVStoreAccessHelper.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.File;
+import java.util.EnumMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+
+/**
+ * KV store test helper
+ *
+ * @author jiachun.fjc
+ */
+public final class KVStoreAccessHelper {
+
+    private static final ExecutorService EXECUTOR = Executors.newSingleThreadExecutor();
+
+    public static void createSstFiles(final RocksRawKVStore store, final EnumMap sstFileTable,
+                                      final byte[] startKey, final byte[] endKey) {
+        store.createSstFiles(sstFileTable, startKey, endKey, EXECUTOR).join();
+    }
+
+    public static void ingestSstFiles(final RocksRawKVStore store, final EnumMap sstFileTable) {
+        store.ingestSstFiles(sstFileTable);
+    }
+
+    public static LocalFileMeta.Builder saveSnapshot(final BaseRawKVStore store, final String snapshotPath,
+                                                     final Region region) throws Exception {
+        final KVStoreSnapshotFile snapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(store);
+        return ((AbstractKVStoreSnapshotFile) snapshotFile).doSnapshotSave(snapshotPath, region, EXECUTOR).get();
+    }
+
+    public static void loadSnapshot(final BaseRawKVStore store, final String snapshotPath, final LocalFileMeta meta,
+                                    final Region region) throws Exception {
+        final KVStoreSnapshotFile snapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(store);
+        ((AbstractKVStoreSnapshotFile) snapshotFile).doSnapshotLoad(snapshotPath, meta, region);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LocalLock.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LocalLock.java
new file mode 100644
index 0000000..00cf12c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LocalLock.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.concurrent.TimeUnit;
+
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+
+/**
+ * for test
+ *
+ * @author jiachun.fjc
+ */
+public class LocalLock extends DistributedLock {
+
+    private final RawKVStore rawKVStore;
+
+    public LocalLock(byte[] target, long lease, TimeUnit unit, RawKVStore rawKVStore) {
+        super(target, lease, unit, null);
+        this.rawKVStore = rawKVStore;
+    }
+
+    @Override
+    public void unlock() {
+        final byte[] internalKey = getInternalKey();
+        final Acquirer acquirer = getAcquirer();
+        this.rawKVStore.releaseLockWith(internalKey, acquirer, null);
+    }
+
+    @Override
+    protected Owner internalTryLock(final byte[] ctx) {
+        final byte[] internalKey = getInternalKey();
+        final Acquirer acquirer = getAcquirer();
+        acquirer.setContext(ctx);
+        final KVStoreClosure closure = new TestClosure();
+        this.rawKVStore.tryLockWith(internalKey, internalKey, false, acquirer, closure);
+        final Owner owner = (Owner) closure.getData();
+        updateOwnerAndAcquirer(owner);
+        return owner;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LongSequenceTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LongSequenceTest.java
new file mode 100644
index 0000000..947adc2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/LongSequenceTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * @author jiachun.fjc
+ */
+public class LongSequenceTest {
+
+    private final AtomicLong index = new AtomicLong();
+
+    @Test
+    public void nextText() {
+        final long base = 100;
+        final LongSequence sequence = new LongSequence(base) {
+
+            @Override
+            public Sequence getNextSequence() {
+                return new Sequence(index.getAndAdd(10), index.get());
+            }
+        };
+        for (int i = 0; i < 100; i++) {
+            Assert.assertTrue(sequence.next() == (long) i + base);
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/SyncKVStore.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/SyncKVStore.java
new file mode 100644
index 0000000..7ba8b83
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/SyncKVStore.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+/**
+ * @author jiachun.fjc
+ */
+public abstract class SyncKVStore {
+
+    @SuppressWarnings("unchecked")
+    public T apply(RawKVStore kvStore) {
+        TestClosure closure = new TestClosure();
+        execute(kvStore, closure);
+        return (T) closure.getData();
+    }
+
+    public abstract void execute(RawKVStore kvStore, KVStoreClosure closure);
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestClosure.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestClosure.java
new file mode 100644
index 0000000..021b5c1
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestClosure.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import com.alipay.sofa.jraft.Status;
+
+/**
+ * @author jiachun.fjc
+ */
+public class TestClosure extends BaseKVStoreClosure {
+
+    @Override
+    public void run(Status status) {
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotReader.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotReader.java
new file mode 100644
index 0000000..8ca2efd
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotReader.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.google.protobuf.Message;
+
+/**
+ * @author jiachun.fjc
+ */
+public class TestSnapshotReader extends SnapshotReader {
+
+    final Map metaTable;
+    final String                                         path;
+
+    public TestSnapshotReader(Map metaTable, String path) {
+        this.metaTable = metaTable;
+        this.path = path;
+    }
+
+    @Override
+    public RaftOutter.SnapshotMeta load() {
+        return null;
+    }
+
+    @Override
+    public String generateURIForCopy() {
+        return null;
+    }
+
+    @Override
+    public boolean init(Void opts) {
+        return false;
+    }
+
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public String getPath() {
+        return this.path;
+    }
+
+    @Override
+    public Set listFiles() {
+        return null;
+    }
+
+    @Override
+    public Message getFileMeta(String fileName) {
+        return this.metaTable.get(fileName);
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotWriter.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotWriter.java
new file mode 100644
index 0000000..71fc9c2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/TestSnapshotWriter.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter;
+import com.alipay.sofa.jraft.entity.RaftOutter;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.google.protobuf.Message;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class TestSnapshotWriter extends SnapshotWriter {
+
+    public final Map metaTable = new ConcurrentHashMap<>();
+    public final String                                         path;
+
+    public TestSnapshotWriter(String path) {
+        this.path = path;
+    }
+
+    @Override
+    public boolean saveMeta(RaftOutter.SnapshotMeta meta) {
+        return false;
+    }
+
+    @Override
+    public boolean addFile(String fileName, Message fileMeta) {
+        final LocalFileMetaOutter.LocalFileMeta.Builder metaBuilder = LocalFileMetaOutter.LocalFileMeta.newBuilder();
+        if (fileMeta != null) {
+            metaBuilder.mergeFrom(fileMeta);
+        }
+        final LocalFileMetaOutter.LocalFileMeta meta = metaBuilder.build();
+        return this.metaTable.putIfAbsent(fileName, meta) == null;
+    }
+
+    @Override
+    public boolean removeFile(String fileName) {
+        return false;
+    }
+
+    @Override
+    public void close(boolean keepDataOnError) throws IOException {
+
+    }
+
+    @Override
+    public boolean init(Void opts) {
+        return false;
+    }
+
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public String getPath() {
+        return this.path;
+    }
+
+    @Override
+    public Set listFiles() {
+        return null;
+    }
+
+    @Override
+    public Message getFileMeta(String fileName) {
+        return this.metaTable.get(fileName);
+    }
+
+    @Override
+    public void close() throws IOException {
+
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/BaseKVStoreTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/BaseKVStoreTest.java
new file mode 100644
index 0000000..8319e57
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/BaseKVStoreTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.memorydb;
+
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore;
+
+public class BaseKVStoreTest {
+
+    protected MemoryRawKVStore kvStore;
+    protected MemoryDBOptions  dbOptions;
+
+    protected void setup() throws Exception {
+        this.kvStore = new MemoryRawKVStore();
+        this.dbOptions = new MemoryDBOptions();
+        this.kvStore.init(this.dbOptions);
+    }
+
+    protected void tearDown() throws Exception {
+        this.kvStore.shutdown();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/MemoryKVStoreTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/MemoryKVStoreTest.java
new file mode 100644
index 0000000..37dbb57
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/memorydb/MemoryKVStoreTest.java
@@ -0,0 +1,867 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.memorydb;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.rhea.StoreEngineHelper;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.options.MemoryDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVIterator;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory;
+import com.alipay.sofa.jraft.rhea.storage.LocalLock;
+import com.alipay.sofa.jraft.rhea.storage.MemoryRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.storage.SyncKVStore;
+import com.alipay.sofa.jraft.rhea.storage.TestClosure;
+import com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader;
+import com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+
+import static com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey;
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * MemoryDB unit test, covering all interfaces in {@link MemoryRawKVStore}
+ *
+ * @author jiachun.fjc
+ */
+public class MemoryKVStoreTest extends BaseKVStoreTest {
+
+    private static final String SNAPSHOT_ARCHIVE = "kv.zip";
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+    }
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#get(byte[], KVStoreClosure)}
+     */
+    @Test
+    public void getTest() {
+        final byte[] key = makeKey("get_test");
+        byte[] value = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertNull(value);
+
+        value = makeValue("get_test_value");
+        this.kvStore.put(key, value, null);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(value, newValue);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#multiGet(List, KVStoreClosure)}
+     */
+    @Test
+    public void multiGetTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("multi_test_key_" + i);
+            byte[] value = makeValue("multi_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        Map mapResult = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.multiGet(keyList, closure);
+            }
+        }.apply(this.kvStore);
+        for (int i = 0; i < keyList.size(); i++) {
+            byte[] key = keyList.get(i);
+            assertArrayEquals(mapResult.get(ByteArray.wrap(key)), valueList.get(i));
+        }
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#localIterator()}
+     */
+    @Test
+    public void getLocalIteratorTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("iterator_test_key_" + i);
+            byte[] value = makeValue("iterator_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+
+        final List entries = Lists.newArrayList();
+        KVIterator it = this.kvStore.localIterator();
+        try {
+            it.seekToFirst();
+            while (it.isValid()) {
+                entries.add(new KVEntry(it.key(), it.value()));
+                it.next();
+            }
+        } finally {
+            try {
+                it.close();
+            } catch (Exception ignored) {
+                // ignored
+            }
+        }
+
+        assertEquals(entries.size(), keyList.size());
+
+        for (int i = 0; i < keyList.size(); i++) {
+            assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+        }
+
+        entries.clear();
+        //
+        it = this.kvStore.localIterator();
+        try {
+            it.seekToLast();
+            while (it.isValid()) {
+                entries.add(new KVEntry(it.key(), it.value()));
+                it.prev();
+            }
+        } finally {
+            try {
+                it.close();
+            } catch (Exception ignored) {
+                // ignored
+            }
+        }
+        for (int i = 0; i < keyList.size(); i++) {
+            final int cmpIndex = keyList.size() - 1 - i;
+            assertArrayEquals(keyList.get(i), entries.get(cmpIndex).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(cmpIndex).getValue());
+        }
+
+        entries.clear();
+        //
+        it = this.kvStore.localIterator();
+        try {
+            it.seek(BytesUtil.writeUtf8("iterator_test_key_5"));
+            while (it.isValid()) {
+                entries.add(new KVEntry(it.key(), it.value()));
+                it.next();
+            }
+        } finally {
+            try {
+                it.close();
+            } catch (Exception ignored) {
+                // ignored
+            }
+        }
+        for (int i = 5; i < keyList.size(); i++) {
+            assertArrayEquals(keyList.get(i), entries.get(i - 5).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(i - 5).getValue());
+        }
+
+        entries.clear();
+        //
+        it = this.kvStore.localIterator();
+        try {
+            it.seekForPrev(BytesUtil.writeUtf8("iterator_test_key_5"));
+            while (it.isValid()) {
+                entries.add(new KVEntry(it.key(), it.value()));
+                it.prev();
+            }
+        } finally {
+            try {
+                it.close();
+            } catch (Exception ignored) {
+                // ignored
+            }
+        }
+        for (int i = 0; i < 5; i++) {
+            assertArrayEquals(keyList.get(i), entries.get(4 - i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(4 - i).getValue());
+        }
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#containsKey(byte[], KVStoreClosure)}
+     */
+    @Test
+    public void containsKeyTest() {
+        final byte[] key = makeKey("contains_key_test");
+        Boolean isContains = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.containsKey(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertFalse(isContains);
+
+        final byte[] value = makeValue("contains_key_test_value");
+        this.kvStore.put(key, value, null);
+        isContains = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.containsKey(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertTrue(isContains);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#scan(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void scanTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            byte[] value = makeValue("scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            byte[] value = makeValue("no_scan_test_value_" + i);
+            this.kvStore.put(key, value, null);
+        }
+        List entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), keyList.size());
+        for (int i = 0; i < keyList.size(); i++) {
+            assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+        }
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(null, null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#reverseScan(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void reverseScanTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            byte[] value = makeValue("scan_test_key_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            byte[] value = makeValue("no_scan_test_key_" + i);
+            this.kvStore.put(key, value, null);
+        }
+
+        List entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), keyList.size());
+        for (int i = keyList.size() - 1; i >= 0; i--) {
+            assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(keyList.size() - 1 - i).getValue());
+        }
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(makeKey("scan_test_key_" + 99), null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(null, null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#getSequence(byte[], int, KVStoreClosure)}
+     */
+    @Test
+    public void getSequenceTest() throws InterruptedException {
+        final byte[] seqKey = makeKey("seq_test");
+        Sequence sequence = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 199, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence.getStartValue(), 0);
+        assertEquals(sequence.getEndValue(), 199);
+
+        Sequence sequence2 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 10, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence2.getStartValue(), 199);
+        assertEquals(sequence2.getEndValue(), 209);
+        this.kvStore.resetSequence(seqKey, null);
+        Sequence sequence3 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 11, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence3.getStartValue(), 0);
+        assertEquals(sequence3.getEndValue(), 11);
+
+        // read-only
+        Sequence sequence4 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 0, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence4.getStartValue(), 11);
+        assertEquals(sequence4.getEndValue(), 11);
+
+        KVStoreClosure assertFailed = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals("Fail to [GET_SEQUENCE], step must >= 0", status.getErrorMsg());
+            }
+        };
+        this.kvStore.getSequence(seqKey, -1, assertFailed);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#getSafeEndValueForSequence(long, int)}
+     */
+    @Test
+    public void getSafeEndValueForSequenceTest() {
+        long startVal = 1;
+        assertEquals(2, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        startVal = Long.MAX_VALUE - 1;
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 2));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, Integer.MAX_VALUE));
+        startVal = Long.MAX_VALUE;
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 0));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 2));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, Integer.MAX_VALUE));
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#put(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void putTest() {
+        final byte[] key = makeKey("put_test");
+        TestClosure closure = new TestClosure();
+        this.kvStore.get(key, closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        this.kvStore.put(key, value, null);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(value, newValue);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#getAndPut(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void getAndPutTest() {
+        final byte[] key = makeKey("put_test");
+        TestClosure closure = new TestClosure();
+        this.kvStore.get(key, closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals(status, Status.OK());
+            }
+        };
+        this.kvStore.getAndPut(key, value, kvStoreClosure);
+        assertNull(kvStoreClosure.getData());
+
+        byte[] newVal = makeValue("put_test_value_new");
+        this.kvStore.getAndPut(key, newVal, kvStoreClosure);
+        assertArrayEquals(value, (byte[]) kvStoreClosure.getData());
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#compareAndPut(byte[], byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void compareAndPutTest() {
+        final byte[] key = makeKey("put_test");
+        byte[] value = makeValue("put_test_value");
+        this.kvStore.put(key, value, null);
+
+        byte[] update = makeValue("put_test_update");
+        KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals(status, Status.OK());
+            }
+        };
+        this.kvStore.compareAndPut(key, value, update, kvStoreClosure);
+        assertEquals(kvStoreClosure.getData(), Boolean.TRUE);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(update, newValue);
+
+        this.kvStore.compareAndPut(key, value, update, kvStoreClosure);
+        assertEquals(kvStoreClosure.getData(), Boolean.FALSE);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#merge(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void mergeTest() {
+        final byte[] key = BytesUtil.writeUtf8("merge_test");
+        final byte[] bytes1 = BytesUtil.writeUtf8("a");
+        final byte[] bytes2 = BytesUtil.writeUtf8("b");
+        final byte[] bytes3 = BytesUtil.writeUtf8("c");
+        final byte[] bytes4 = BytesUtil.writeUtf8("d");
+        this.kvStore.put(key, bytes1, null);
+        this.kvStore.merge(key, bytes2, null);
+        this.kvStore.merge(key, bytes3, null);
+        this.kvStore.merge(key, bytes4, null);
+
+        final byte[] val = new SyncKVStore() {
+
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(BytesUtil.writeUtf8("a,b,c,d"), val);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#put(List, KVStoreClosure)}
+     */
+    @Test
+    public void putListTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("batch_put_test_key" + i), makeValue("batch_put_test_value" + i)));
+        }
+        this.kvStore.put(entries, null);
+        final List entries2 = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(makeKey("batch_put_test_key"), makeKey("batch_put_test_key" + 99), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), entries2.size());
+        for (int i = 0; i < entries.size(); i++) {
+            assertArrayEquals(entries.get(i).getKey(), entries2.get(i).getKey());
+            assertArrayEquals(entries.get(i).getValue(), entries2.get(i).getValue());
+        }
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#putIfAbsent(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void putIfAbsent() {
+        byte[] key = makeKey("put_if_absent_test");
+        byte[] value = makeValue("put_if_absent_test_value");
+        TestClosure closure = new TestClosure();
+        this.kvStore.putIfAbsent(key, value, closure);
+        assertNull(closure.getData());
+        this.kvStore.putIfAbsent(key, value, closure);
+        assertArrayEquals(value, (byte[]) closure.getData());
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#tryLockWith(byte[], byte[], boolean, DistributedLock.Acquirer, KVStoreClosure)}
+     */
+    @Test
+    public void tryLockWith() throws InterruptedException {
+        byte[] lockKey = makeKey("lock_test");
+        final DistributedLock lock = new LocalLock(lockKey, 3, TimeUnit.SECONDS, this.kvStore);
+        assertNotNull(lock);
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock(3001, TimeUnit.MILLISECONDS));
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            final DistributedLock lock2 = new LocalLock(lockKey, 3, TimeUnit.SECONDS, this.kvStore);
+            try {
+                assertTrue(!lock2.tryLock());
+            } finally {
+                latch.countDown();
+            }
+        }, "lock1-thread").start();
+        latch.await();
+        lock.unlock();
+        assertTrue(lock.tryLock());
+        lock.unlock();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("del_test" + i), makeValue("del_test_value")));
+        }
+        this.kvStore.put(entries, null);
+        this.kvStore.delete(makeKey("del_test5"), null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("del_test"), makeKey("del_test" + 99), closure);
+        List entries2 = (List) closure.getData();
+        assertEquals(entries.size() - 1, entries2.size());
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_test5"), closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteRangeTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("del_range_test" + i), makeValue("del_range_test" + i)));
+        }
+        this.kvStore.put(entries, null);
+        // delete [del_range_test5, del_range_test8)
+        this.kvStore.deleteRange(makeKey("del_range_test5"), makeKey("del_range_test8"), null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("del_range_test"), makeKey("del_range_test" + 99), closure);
+        final List entries2 = (List) closure.getData();
+        assertEquals(entries.size() - 3, entries2.size());
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test5"), closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test6"), closure);
+        value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test7"), closure);
+        value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test8"), closure);
+        value = (byte[]) closure.getData();
+        assertNotNull(value);
+    }
+
+    /**
+     * Test method: {@link MemoryRawKVStore#delete(List, KVStoreClosure)}
+     */
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteListTest() {
+        final List entries = Lists.newArrayList();
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] key = makeKey("batch_del_test" + i);
+            entries.add(new KVEntry(key, makeValue("batch_del_test_value")));
+            keys.add(key);
+        }
+        this.kvStore.put(entries, null);
+        this.kvStore.delete(keys, null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("batch_del_test"), makeKey("batch_del_test" + 99), closure);
+        List entries2 = (List) closure.getData();
+        assertEquals(0, entries2.size());
+        for (int i = 0; i < keys.size(); i++) {
+            closure = new TestClosure();
+            this.kvStore.get(keys.get(i), closure);
+            byte[] value = (byte[]) closure.getData();
+            assertNull(value);
+        }
+    }
+
+    private byte[] get(final byte[] key) {
+        return new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+    }
+
+    @Test
+    public void snapshotTest() throws Exception {
+        final File backupDir = new File("backup");
+        if (backupDir.exists()) {
+            FileUtils.deleteDirectory(backupDir);
+        }
+        FileUtils.forceMkdir(backupDir);
+        for (int i = 0; i < 100000; i++) {
+            final String v = String.valueOf(i);
+            this.kvStore.put(makeKey(v), makeValue(v), null);
+        }
+        for (int i = 0; i < 10000; i++) {
+            this.kvStore.getSequence(makeKey((i % 100) + "seq_test"), 10, null);
+        }
+        final Region region = new Region();
+        KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
+        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Closure done = status -> {
+            assertTrue(status.isOk());
+            latch.countDown();
+        };
+        kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
+        latch.await();
+        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
+        assertNotNull(meta);
+
+        assertNotNull(get(makeKey("1")));
+
+        this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
+        assertNotNull(get(makeKey("100001")));
+
+        this.kvStore.shutdown();
+        this.kvStore = new MemoryRawKVStore();
+        final MemoryDBOptions dbOpts = new MemoryDBOptions();
+        this.kvStore.init(dbOpts);
+
+        assertNull(get(makeKey("1")));
+
+        final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
+        kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
+        assertTrue(ret);
+
+
+        for (int i = 0; i < 100000; i++) {
+            final String v = String.valueOf(i);
+            assertArrayEquals(makeValue(v), get(makeKey(v)));
+        }
+
+        // Key[100001] is put after the snapshot, so key[100001] should not exist
+        assertNull(get(makeKey("100001")));
+
+        FileUtils.deleteDirectory(backupDir);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
+    }
+
+    @Test
+    public void multiGroupSnapshotTest() throws Exception {
+        final File backupDir = new File("multi-backup");
+        if (backupDir.exists()) {
+            FileUtils.deleteDirectory(backupDir);
+        }
+
+        final List regions = Lists.newArrayList();
+        regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
+        regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
+        regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
+        regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
+        regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
+
+        for (int i = 0; i < 5; i++) {
+            final String v = String.valueOf(i);
+            this.kvStore.put(makeKey(v), makeValue(v), null);
+        }
+        for (int i = 0; i < 5; i++) {
+            this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
+        }
+
+        KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
+        final List writers = Lists.newArrayList();
+        for (int i = 0; i < 4; i++) {
+            final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
+            final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
+            writers.add(snapshotWriter);
+            final CountDownLatch latch = new CountDownLatch(1);
+            final Closure done = status -> {
+                assertTrue(status.isOk());
+                latch.countDown();
+            };
+            kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
+            latch.await();
+            final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
+            assertNotNull(meta);
+        }
+
+        this.kvStore.shutdown();
+        this.kvStore = new MemoryRawKVStore();
+        final MemoryDBOptions dbOpts = new MemoryDBOptions();
+        this.kvStore.init(dbOpts);
+
+        kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        for (int i = 0; i < 4; i++) {
+            final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
+            final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
+            final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
+            assertTrue(ret);
+        }
+
+        for (int i = 0; i < 4; i++) {
+            final String v = String.valueOf(i);
+            final byte[] seqKey = makeKey(i + "_seq_test");
+            assertArrayEquals(makeValue(v), get(makeKey(v)));
+            final Sequence sequence = new SyncKVStore() {
+                @Override
+                public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                    kvStore.getSequence(seqKey, 10, closure);
+                }
+            }.apply(this.kvStore);
+            assertEquals(10L, sequence.getStartValue());
+            assertEquals(20L, sequence.getEndValue());
+        }
+
+        assertNull(get(makeKey("5")));
+        final Sequence sequence = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(0L, sequence.getStartValue());
+
+        FileUtils.deleteDirectory(backupDir);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
+    }
+
+    @Test
+    public void getApproximateKeysInRangeTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10000; i++) {
+            entries.add(new KVEntry(makeKey("approximate_test" + i), makeValue("approximate_test_value")));
+        }
+        this.kvStore.put(entries, null);
+
+        long approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test" + 9999), null);
+        assertEquals(1, approximateKeys);
+        approximateKeys = this.kvStore.getApproximateKeysInRange(null, makeKey("approximate_test" + 9999));
+        assertEquals(10000, approximateKeys, 1);
+        approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test" + 9990),
+            makeKey("approximate_test" + 9999));
+        assertEquals(10, approximateKeys, 1);
+    }
+
+    @Test
+    public void jumpOverTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10000; i++) {
+            entries.add(new KVEntry(makeKey("approximate_test" + i), makeValue("approximate_test_value")));
+        }
+        this.kvStore.put(entries, null);
+        final byte[] endKey = this.kvStore.jumpOver(makeKey("approximate_test0000"), 1000);
+        final long approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test0000"), endKey);
+        assertEquals(1000, approximateKeys, 10);
+    }
+
+    @Test
+    public void initFencingTokenTest() throws Exception {
+        final Method getNextFencingMethod = MemoryRawKVStore.class.getDeclaredMethod("getNextFencingToken",
+            byte[].class);
+        getNextFencingMethod.setAccessible(true);
+        final List parentKeys = Lists.newArrayList();
+        parentKeys.add(null); // startKey == null
+        parentKeys.add(BytesUtil.writeUtf8("parent"));
+        for (int i = 0; i < 2; i++) {
+            final byte[] parentKey = parentKeys.get(i);
+            final byte[] childKey = BytesUtil.writeUtf8("child");
+            assertEquals(1L, getNextFencingMethod.invoke(this.kvStore, (Object) parentKey));
+            assertEquals(2L, getNextFencingMethod.invoke(this.kvStore, (Object) parentKey));
+            this.kvStore.initFencingToken(parentKey, childKey);
+            assertEquals(3L, getNextFencingMethod.invoke(this.kvStore, (Object) childKey));
+            assertEquals(4L, getNextFencingMethod.invoke(this.kvStore, (Object) childKey));
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractDistributedLockTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractDistributedLockTest.java
new file mode 100644
index 0000000..291fb68
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractDistributedLockTest.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.errors.InvalidLockAcquirerException;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public abstract class AbstractDistributedLockTest extends RheaKVTestCluster {
+
+    private static final ExecutorService EXECUTOR = Executors.newFixedThreadPool(4);
+
+    @Rule
+    public TestName                      testName = new TestName();
+
+    public abstract StorageType getStorageType();
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+        super.start(getStorageType());
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.shutdown();
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+    }
+
+    private void lockTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "lockTest";
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        boolean locked = lock.tryLock();
+        assertTrue(locked);
+        Future f = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Assert.assertTrue(!f.get());
+        lock.unlock();
+        f = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Assert.assertTrue(f.get());
+    }
+
+    private void lockKeepLeaseTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "lockKeepLeaseTest";
+        final ScheduledExecutorService watchdog = Executors.newScheduledThreadPool(2);
+        final DistributedLock lock = store.getDistributedLock(lockKey, 1, TimeUnit.SECONDS, watchdog);
+        assertNotNull(lock);
+        boolean locked = lock.tryLock();
+        assertTrue(locked);
+        Thread.sleep(2000);
+        Future f = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Assert.assertTrue(!f.get());
+        lock.unlock();
+        f = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Assert.assertTrue(f.get());
+        ExecutorServiceHelper.shutdownAndAwaitTermination(watchdog);
+    }
+
+    private void reentrantLockTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "reentrantLockTest";
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        for (int i = 0; i < 5; i++) {
+            boolean locked = lock.tryLock();
+            assertTrue(locked);
+            assertEquals(i + 1L, lock.getOwner().getAcquires());
+        }
+        for (int i = 5; i > 0; i--) {
+            lock.unlock();
+        }
+        lock.tryLock();
+        assertEquals(1L, lock.getOwner().getAcquires());
+    }
+
+    private void lockFencingTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "lockFencingTest";
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        long prevToken = 0;
+        for (int i = 0; i < 5; i++) {
+            boolean locked = lock.tryLock();
+            assertTrue(locked);
+            final long token = lock.getFencingToken();
+            System.err.println("token=" + token);
+            assertTrue(token > prevToken);
+            prevToken = token;
+            lock.unlock();
+        }
+    }
+
+    private void invalidUnlockTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "invalidUnlock";
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        assertTrue(lock.tryLock());
+        final DistributedLock anotherLock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        try {
+            anotherLock.unlock();
+            assertTrue(false);
+        } catch (final Exception e) {
+            e.printStackTrace();
+            assertTrue(e instanceof InvalidLockAcquirerException);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private void concurrentLockingTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "concurrentLockingTest";
+        Future f1 = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Future f2 = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Future f3 = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        Future f4 = EXECUTOR.submit(() -> store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS).tryLock());
+        assertTrue(f1.get() || f2.get() || f3.get() || f4.get());
+        assertTrue(!(f1.get() && f2.get() && f3.get() && f4.get()));
+        int count = 0;
+        for (Future f : new Future[] { f1, f2, f3, f4}) {
+            if (f.get()) {
+                count++;
+            }
+        }
+        assertEquals(1, count);
+    }
+
+    private void lockCtxTest(final RheaKVStore store) throws Exception {
+        final String lockKey = "lockCtxTest";
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        final byte[] ctx1 = BytesUtil.writeUtf8("first");
+        assertTrue(lock.tryLock(ctx1));
+        assertArrayEquals(ctx1, lock.getOwnerContext());
+
+        final DistributedLock anotherLock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(anotherLock);
+        final byte[] ctx2 = BytesUtil.writeUtf8("second");
+        assertTrue(!anotherLock.tryLock(ctx2));
+        assertArrayEquals(ctx1, anotherLock.getOwnerContext());
+        assertEquals(lock.getAcquirer().getId(), anotherLock.getOwner().getId());
+    }
+
+    @Test
+    public void lockByLeaderTest() throws Exception {
+        lockTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void lockByFollowerTest() throws Exception {
+        lockTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void lockKeepLeaseByLeaderTest() throws Exception {
+        lockKeepLeaseTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void lockKeepLeaseByFollowerTest() throws Exception {
+        lockKeepLeaseTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void reentrantLockByLeaderTest() throws Exception {
+        reentrantLockTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void reentrantLockByFollowerTest() throws Exception {
+        reentrantLockTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void lockFencingByLeaderTest() throws Exception {
+        lockFencingTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void lockFencingByFollowerTest() throws Exception {
+        lockFencingTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void invalidUnlockByLeaderTest() throws Exception {
+        invalidUnlockTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void invalidUnlockByFollowerTest() throws Exception {
+        invalidUnlockTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void concurrentLockingByLeaderTest() throws Exception {
+        concurrentLockingTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void concurrentLockingByFollowerTest() throws Exception {
+        concurrentLockingTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void lockCtxByLeaderTest() throws Exception {
+        lockCtxTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void lockCtxByFollowerTest() throws Exception {
+        lockCtxTest(getRandomFollowerStore());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractRheaKVStoreTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractRheaKVStoreTest.java
new file mode 100644
index 0000000..7b8503f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/AbstractRheaKVStoreTest.java
@@ -0,0 +1,1207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rhea.JRaftHelper;
+import com.alipay.sofa.jraft.rhea.RheaKVServiceFactory;
+import com.alipay.sofa.jraft.rhea.client.FutureGroup;
+import com.alipay.sofa.jraft.rhea.client.RheaIterator;
+import com.alipay.sofa.jraft.rhea.client.RheaKVCliService;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.storage.CASEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey;
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * RheaKVStore unit test, covering all interfaces of {@link RheaKVStore}
+ *
+ * @author jiachun.fjc
+ */
+public abstract class AbstractRheaKVStoreTest extends RheaKVTestCluster {
+
+    public abstract StorageType getStorageType();
+
+    @Rule
+    public TestName testName = new TestName();
+
+    @Before
+    public void setup() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Start test method: " + this.testName.getMethodName());
+        super.start(getStorageType());
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        System.out.println(">>>>>>>>>>>>>>> Stopping test method: " + this.testName.getMethodName());
+        super.shutdown();
+        System.out.println(">>>>>>>>>>>>>>> End test method: " + this.testName.getMethodName());
+    }
+
+    private void checkRegion(RheaKVStore store, byte[] key, long expectedRegionId) {
+        Region region = store.getPlacementDriverClient().findRegionByKey(key, false);
+        assertEquals(expectedRegionId, region.getId());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#get(byte[])}
+     */
+    private void getTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] key = makeKey("a_get_test");
+        checkRegion(store, key, 1);
+        byte[] value = store.bGet(key);
+        assertNull(value);
+        value = makeValue("a_get_test_value");
+        store.bPut(key, value);
+        assertArrayEquals(value, store.bGet(key));
+
+        key = makeKey("h_get_test");
+        checkRegion(store, key, 2);
+        value = store.bGet(key);
+        assertNull(value);
+        value = makeValue("h_get_test_value");
+        store.bPut(key, value);
+        assertArrayEquals(value, store.bGet(key));
+
+        key = makeKey("z_get_test");
+        checkRegion(store, key, 2);
+        value = store.bGet(key);
+        assertNull(value);
+        value = makeValue("z_get_test_value");
+        store.bPut(key, value);
+        assertArrayEquals(value, store.bGet(key));
+    }
+
+    @Test
+    public void getByLeaderTest() {
+        getTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void getByFollowerTest() {
+        getTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void putByLeaderGetByFollower() {
+        byte[] key = makeKey("get_test");
+        byte[] value = getRandomLeaderStore().bGet(key);
+        assertNull(value);
+        value = makeValue("get_test_value");
+        getRandomLeaderStore().bPut(key, value);
+        assertArrayEquals(value, getRandomFollowerStore().bGet(key));
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#multiGet(List)}
+     */
+    private void multiGetTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List keyList = Lists.newArrayList();
+        List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("a_multi_test_key_" + i);
+            checkRegion(store, key, 1);
+            byte[] value = makeValue("a_multi_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("j_multi_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("j_multi_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("t_multi_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("t_multi_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        keyList.add(makeKey("non_existing_key"));
+        Map mapResult = store.bMultiGet(keyList);
+        for (int i = 0; i < valueList.size(); i++) {
+            byte[] key = keyList.get(i);
+            byte[] value = mapResult.get(ByteArray.wrap(key));
+            assertArrayEquals(value, valueList.get(i));
+        }
+        byte[] lastKey = keyList.get(keyList.size() - 1);
+        byte[] value = mapResult.get(ByteArray.wrap(lastKey));
+        assertNull(value);
+    }
+
+    @Test
+    public void multiGetByLeaderTest() {
+        multiGetTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void multiGetByFollowerTest() {
+        multiGetTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#containsKey(byte[])}
+     */
+    private void containsKeyTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] key = makeKey("a_contains_key_test");
+        checkRegion(store, key, 1);
+        Boolean isContains = store.bContainsKey(key);
+        assertFalse(isContains);
+        byte[] value = makeValue("a_contains_key_test_value");
+        store.bPut(key, value);
+        assertTrue(store.bContainsKey(key));
+
+        key = makeKey("h_contains_key_test");
+        checkRegion(store, key, 2);
+        isContains = store.bContainsKey(key);
+        assertFalse(isContains);
+        value = makeValue("h_contains_key_test_value");
+        store.bPut(key, value);
+        assertTrue(store.bContainsKey(key));
+
+        key = makeKey("z_contains_key_test");
+        checkRegion(store, key, 2);
+        isContains = store.bContainsKey(key);
+        assertFalse(isContains);
+        value = makeValue("z_contains_key_test_value");
+        store.bPut(key, value);
+        assertTrue(store.bContainsKey(key));
+    }
+
+    @Test
+    public void containsKeyByLeaderTest() {
+        containsKeyTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void containsKeyByFollowerTest() {
+        containsKeyTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void scanByLeaderTest() {
+        scanTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void scanByFollowerTest() {
+        scanTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#scan(byte[], byte[])}
+     */
+    private void scanTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List keyList = Lists.newArrayList();
+        List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("no_scan_test_value_" + i);
+            store.bPut(key, value);
+        }
+
+        // bScan(byte[], byte[])
+        {
+            List entries = store.bScan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99));
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+        }
+
+        // bScan(String, String)
+        {
+            List entries = store.bScan("scan_test_key_", "scan_test_key_" + 99);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+        }
+
+        // bScan(byte[], byte[], Boolean)
+        {
+            List entries = store.bScan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99), true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+        }
+
+        // bScan(String, String, Boolean)
+        {
+            List entries = store.bScan("scan_test_key_", "scan_test_key_" + 99, true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+        }
+
+        // bScan(byte[], byte[], Boolean, Boolean)
+        {
+            List entries = store.bScan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99), true, true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+
+            entries = store.bScan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99), true, false);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertNull(entries.get(i).getValue());
+            }
+        }
+
+        // bScan(String, String, Boolean, Boolean)
+        {
+            List entries = store.bScan("scan_test_key_", "scan_test_key_" + 99, true, true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+            }
+
+            entries = store.bScan("scan_test_key_", "scan_test_key_" + 99, true, false);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+                assertNull(entries.get(i).getValue());
+            }
+        }
+
+        {
+            List entries = store.bScan(null, makeKey("no_scan_test_key_" + 99));
+            assertEquals(entries.size(), keyList.size());
+
+            entries = store.bScan("no_", null);
+            assertEquals(entries.size(), 20);
+        }
+
+        {
+            List entries = store.bScan("z", null);
+            assertEquals(entries.size(), 0);
+        }
+    }
+
+    @Test
+    public void reverseScanByLeaderTest() {
+        reverseScanTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void reverseScanByFollowerTest() {
+        reverseScanTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#reverseScan(byte[], byte[])}
+     */
+    private void reverseScanTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List keyList = Lists.newArrayList();
+        List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("no_scan_test_value_" + i);
+            store.bPut(key, value);
+        }
+
+        // bReverseScan(byte[], byte[])
+        {
+            List entries = store.bReverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"));
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+        }
+
+        // bReverseScan(String, String)
+        {
+            List entries = store.bReverseScan("scan_test_key_" + 99, "scan_test_key_");
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+        }
+
+        // bReverseScan(byte[], byte[], Boolean)
+        {
+            List entries = store.bReverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"), true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+        }
+
+        // bReverseScan(String, String, Boolean)
+        {
+            List entries = store.bReverseScan("scan_test_key_" + 99, "scan_test_key_", true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+        }
+
+        // bReverseScan(byte[], byte[], Boolean, Boolean)
+        {
+            List entries = store.bReverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"), true,
+                true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+
+            entries = store.bReverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"), true, false);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertNull(entries.get(i).getValue());
+            }
+        }
+
+        // bReverseScan(String, String, Boolean, Boolean)
+        {
+            List entries = store.bReverseScan("scan_test_key_" + 99, "scan_test_key_", true, true);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertArrayEquals(valueList.get(i), entries.get(valueList.size() - 1 - i).getValue());
+            }
+
+            entries = store.bReverseScan("scan_test_key_" + 99, "scan_test_key_", true, false);
+            assertEquals(entries.size(), keyList.size());
+            for (int i = 0; i < keyList.size(); i++) {
+                assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+                assertNull(entries.get(i).getValue());
+            }
+        }
+
+        {
+            List entries = store.bReverseScan(makeKey("no_scan_test_key_" + 99), null);
+            assertEquals(entries.size(), keyList.size());
+
+            entries = store.bReverseScan(null, "no_");
+            assertEquals(entries.size(), 20);
+
+        }
+
+        {
+            List entries = store.bReverseScan(null, "z");
+            assertEquals(entries.size(), 0);
+        }
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#scan(byte[], byte[])}
+     */
+    private void iteratorTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List keyList = Lists.newArrayList();
+        List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("a_iterator_test_key_" + i);
+            checkRegion(store, key, 1);
+            byte[] value = makeValue("iterator_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("h_iterator_test_key_" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("iterator_scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            store.bPut(key, value);
+        }
+
+        // iterator(byte[], byte[], int)
+        {
+            RheaIterator iterator = store.iterator(makeKey("a_iterator_test_key_"), makeKey("z"), 2);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertArrayEquals(valueList.get(index), kvEntry.getValue());
+            }
+        }
+
+        // iterator(String, String, int)
+        {
+            RheaIterator iterator = store.iterator("a_iterator_test_key_", "z", 2);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertArrayEquals(valueList.get(index), kvEntry.getValue());
+            }
+        }
+
+        // iterator(byte[], byte[], int, boolean, boolean)
+        {
+            RheaIterator iterator = store.iterator(makeKey("a_iterator_test_key_"), makeKey("z"), 2, true,
+                false);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertNull(kvEntry.getValue());
+            }
+        }
+
+        // iterator(String, String, int, boolean, boolean)
+        {
+            RheaIterator iterator = store.iterator("a_iterator_test_key_", "z", 2, true, false);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertNull(kvEntry.getValue());
+            }
+        }
+
+        {
+            RheaIterator iterator = store.iterator("a_iterator_test_key_", "z", 2);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertArrayEquals(valueList.get(index), kvEntry.getValue());
+            }
+        }
+
+        {
+            RheaIterator iterator = store.iterator("a", null, 5);
+            int i = 0;
+            while (iterator.hasNext()) {
+                final int index = i++;
+                final KVEntry kvEntry = iterator.next();
+                System.err.println("index " + index);
+                System.err.println(BytesUtil.readUtf8(kvEntry.getKey()));
+                assertArrayEquals(keyList.get(index), kvEntry.getKey());
+                assertArrayEquals(valueList.get(index), kvEntry.getValue());
+            }
+        }
+    }
+
+    @Test
+    public void iteratorByLeaderTest() {
+        iteratorTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void iteratorByFollowerTest() {
+        iteratorTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#getSequence(byte[], int)}
+     */
+    private void getSequenceTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] seqKey = makeKey("seq_test");
+        checkRegion(store, seqKey, 2);
+        Sequence sequence = store.bGetSequence(seqKey, 199);
+        assertEquals(sequence.getStartValue(), 0);
+        assertEquals(sequence.getEndValue(), 199);
+
+        // get-only, do not update
+        final long latestVal = store.bGetLatestSequence(seqKey);
+        assertEquals(latestVal, 199);
+
+        sequence = store.bGetSequence(seqKey, 10);
+        assertEquals(sequence.getStartValue(), 199);
+        assertEquals(sequence.getEndValue(), 209);
+
+        sequence = store.bGetSequence(seqKey, 10);
+        assertEquals(sequence.getStartValue(), 209);
+        assertEquals(sequence.getEndValue(), 219);
+
+        assertNull(store.bGet(seqKey));
+        store.bResetSequence(seqKey);
+        sequence = store.bGetSequence(seqKey, 11);
+        assertEquals(sequence.getStartValue(), 0);
+        assertEquals(sequence.getEndValue(), 11);
+    }
+
+    @Test
+    public void getSequenceByLeaderTest() {
+        getSequenceTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void getSequenceByFollowerTest() {
+        getSequenceTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#put(byte[], byte[])}
+     */
+    private void putTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] key = makeKey("put_test");
+        checkRegion(store, key, 2);
+        byte[] value = store.bGet(key);
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        store.bPut(key, value);
+        byte[] newValue = store.bGet(key);
+        assertArrayEquals(value, newValue);
+    }
+
+    @Test
+    public void putByLeaderTest() {
+        putTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void putByFollowerTest() {
+        putTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#getAndPut(byte[], byte[])}
+     */
+    private void getAndPutTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] key = makeKey("put_test");
+        checkRegion(store, key, 2);
+        byte[] value = store.bGet(key);
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        byte[] preVal = store.bGetAndPut(key, value);
+        assertNull(preVal);
+        byte[] newVal = makeValue("put_test_value_new");
+        preVal = store.bGetAndPut(key, newVal);
+        assertArrayEquals(value, preVal);
+    }
+
+    @Test
+    public void getAndPutByLeaderTest() {
+        getAndPutTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void getAndPutByFollowerTest() {
+        getAndPutTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#compareAndPut(byte[], byte[], byte[])}
+     */
+    private void compareAndPutTest(RheaKVStore store) {
+        byte[] key = makeKey("put_test");
+        checkRegion(store, key, 2);
+        byte[] value = makeValue("put_test_value");
+        store.bPut(key, value);
+
+        byte[] update = makeValue("put_test_update");
+        assertTrue(store.bCompareAndPut(key, value, update));
+        byte[] newValue = store.bGet(key);
+        assertArrayEquals(update, newValue);
+
+        assertFalse(store.bCompareAndPut(key, value, update));
+    }
+
+    @Test
+    public void compareAndPutByLeaderTest() {
+        compareAndPutTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void compareAndPutByFollowerTest() {
+        compareAndPutTest(getRandomFollowerStore());
+    }
+
+    /**
+     *
+     * Test method: {@link RheaKVStore#compareAndPutAll(List)}
+     */
+    public void compareAndPutAllTest(final RheaKVStore store) {
+        final List entries = new ArrayList<>();
+        entries.add(new CASEntry(makeKey("k1"), null, makeValue("v1")));
+        entries.add(new CASEntry(makeKey("k2"), null, makeValue("v2")));
+        entries.add(new CASEntry(makeKey("k3"), null, makeValue("v3")));
+
+        boolean ret = store.bCompareAndPutAll(entries);
+        assertTrue(ret);
+
+        entries.clear();
+        entries.add(new CASEntry(makeKey("k1"), makeValue("v1"), makeValue("v11")));
+        entries.add(new CASEntry(makeKey("k2"), makeValue("v2"), makeValue("v22")));
+
+        ret = store.bCompareAndPutAll(entries);
+        assertTrue(ret);
+
+        entries.clear();
+        entries.add(new CASEntry(makeKey("k1"), makeValue("v11"), makeValue("v111")));
+        entries.add(new CASEntry(makeKey("k2"), makeValue("v22"), makeValue("v222")));
+        entries.add(new CASEntry(makeKey("k3"), makeValue("v33"), makeValue("v333")));
+
+        ret = store.bCompareAndPutAll(entries);
+        assertTrue(!ret);
+
+        final Map map = store.bMultiGet(Lists.newArrayList(makeKey("k1"), makeKey("k2"),
+            makeKey("k3")));
+        assertArrayEquals(makeValue("v11"), map.get(ByteArray.wrap(makeKey("k1"))));
+        assertArrayEquals(makeValue("v22"), map.get(ByteArray.wrap(makeKey("k2"))));
+        assertArrayEquals(makeValue("v3"), map.get(ByteArray.wrap(makeKey("k3"))));
+    }
+
+    @Test
+    public void compareAndPutAllByLeaderTest() {
+        compareAndPutAllTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void compareAndPutAllByFollowerTest() {
+        compareAndPutAllTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#merge(String, String)}
+     */
+    private void mergeTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, t), 3 -> [t, null)
+        String key = "key";
+        byte[] value = store.bGet(key);
+        assertNull(value);
+
+        store.bMerge(key, "aa");
+        store.bMerge(key, "bb");
+        store.bMerge(key, "cc");
+        value = store.bGet(key);
+        assertArrayEquals(value, BytesUtil.writeUtf8("aa,bb,cc"));
+    }
+
+    @Test
+    public void mergeByLeaderTest() {
+        mergeTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void mergeByFollowerTest() {
+        mergeTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#put(List)}
+     */
+    private void putListTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List entries1 = Lists.newArrayList();
+        for (int i = 0; i < 3; i++) {
+            byte[] key = makeKey("batch_put_test_key" + i);
+            checkRegion(store, key, 1);
+            entries1.add(new KVEntry(key, makeValue("batch_put_test_value" + i)));
+        }
+        store.bPut(entries1);
+        List entries2 = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("g_batch_put_test_key" + i);
+            checkRegion(store, key, 2);
+            entries2.add(new KVEntry(key, makeValue("batch_put_test_value" + i)));
+        }
+        store.bPut(entries2);
+        List foundList = store.bScan(makeKey("batch_put_test_key"), makeKey("batch_put_test_key" + 99));
+        assertEquals(entries1.size(), foundList.size());
+        for (int i = 0; i < entries1.size(); i++) {
+            assertArrayEquals(entries1.get(i).getKey(), foundList.get(i).getKey());
+            assertArrayEquals(entries1.get(i).getValue(), foundList.get(i).getValue());
+        }
+        foundList = store.bScan(makeKey("g_batch_put_test_key"), makeKey("g_batch_put_test_key" + 99));
+        assertEquals(entries2.size(), foundList.size());
+        for (int i = 0; i < entries2.size(); i++) {
+            assertArrayEquals(entries2.get(i).getKey(), foundList.get(i).getKey());
+            assertArrayEquals(entries2.get(i).getValue(), foundList.get(i).getValue());
+        }
+    }
+
+    @Test
+    public void putListByLeaderTest() {
+        putListTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void putListByFollowerTest1() {
+        putListTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#putIfAbsent(byte[], byte[])}
+     */
+    private void putIfAbsentTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] key = makeKey("u_put_if_absent_test");
+        checkRegion(store, key, 2);
+        byte[] value = makeValue("put_if_absent_test_value");
+        byte[] newValue = makeValue("put_if_absent_test_value_1");
+        byte[] oldValue = store.bPutIfAbsent(key, value);
+        assertNull(oldValue);
+        oldValue = store.bPutIfAbsent(key, newValue);
+        assertArrayEquals(value, oldValue);
+    }
+
+    @Test
+    public void putIfAbsentByLeaderTest() {
+        putIfAbsentTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void putIfAbsentByFollowerTest() {
+        putIfAbsentTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#delete(byte[])}
+     */
+    private void deleteTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("del_test" + i);
+            checkRegion(store, key, 1);
+            byte[] value = makeValue("del_test_value" + i);
+            entries.add(new KVEntry(key, value));
+        }
+        store.bPut(entries);
+        store.bDelete(makeKey("del_test5"));
+        List entries2 = store.bScan(makeKey("del_test"), makeKey("del_test" + 99));
+        assertEquals(entries.size() - 1, entries2.size());
+        byte[] value = store.bGet(makeKey("del_test5"));
+        assertNull(value);
+    }
+
+    @Test
+    public void deleteByLeaderTest() {
+        deleteTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void deleteByFollowerTest1() {
+        deleteTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#deleteRange(byte[], byte[])}
+     */
+    private void deleteRangeTest(RheaKVStore store) {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("del_range_test" + i);
+            checkRegion(store, key, 1);
+            byte[] value = makeValue("del_range_test_value" + i);
+            entries.add(new KVEntry(key, value));
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("t_del_range_test" + i);
+            checkRegion(store, key, 2);
+            byte[] value = makeValue("del_range_test_value" + i);
+            entries.add(new KVEntry(key, value));
+        }
+        store.bPut(entries);
+        // delete [del_range_test5, t_del_range_test8)
+        // cover three region: 1 -> [null, g), 2 -> [g, t), 3 -> [t, null)
+        store.bDeleteRange(makeKey("del_range_test5"), makeKey("t_del_range_test8"));
+        List entries2 = store.bScan(makeKey("del_range_test"), makeKey("t_del_range_test" + 99));
+        assertEquals(entries.size() - 13, entries2.size());
+        byte[] value = store.bGet(makeKey("del_range_test5"));
+        assertNull(value);
+        value = store.bGet(makeKey("del_range_test6"));
+        assertNull(value);
+        value = store.bGet(makeKey("del_range_test7"));
+        assertNull(value);
+        value = store.bGet(makeKey("t_del_range_test8"));
+        assertNotNull(value);
+    }
+
+    @Test
+    public void deleteRangeByLeaderTest() {
+        deleteRangeTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void deleteRangeByFollowerTest() {
+        deleteRangeTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#delete(List)}
+     */
+    private void deleteListTest(RheaKVStore store) {
+        List entries1 = Lists.newArrayList();
+        List keys1 = Lists.newArrayList();
+        for (int i = 0; i < 3; i++) {
+            byte[] key = makeKey("batch_del_test_key" + i);
+            checkRegion(store, key, 1);
+            entries1.add(new KVEntry(key, makeValue("batch_del_test_value" + i)));
+            keys1.add(key);
+        }
+        store.bPut(entries1);
+        store.bDelete(keys1);
+        List entries2 = Lists.newArrayList();
+        List keys2 = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("g_batch_del_test_key" + i);
+            checkRegion(store, key, 2);
+            entries2.add(new KVEntry(key, makeValue("batch_del_test_value" + i)));
+            keys2.add(key);
+        }
+        store.bPut(entries2);
+        store.bDelete(keys2);
+        List foundList = store.bScan(makeKey("batch_del_test_key"), makeKey("batch_del_test_key" + 99));
+        assertEquals(0, foundList.size());
+        for (int i = 0; i < keys1.size(); i++) {
+            byte[] value = store.bGet(keys1.get(i));
+            assertNull(value);
+        }
+        foundList = store.bScan(makeKey("g_batch_del_test_key"), makeKey("g_batch_put_test_key" + 99));
+        assertEquals(0, foundList.size());
+        for (int i = 0; i < keys2.size(); i++) {
+            byte[] value = store.bGet(keys2.get(i));
+            assertNull(value);
+        }
+    }
+
+    @Test
+    public void deleteListByLeaderTest() {
+        deleteListTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void deleteListByFollowerTest() {
+        deleteListTest(getRandomFollowerStore());
+    }
+
+    /**
+     * Test method: {@link RheaKVStore#getDistributedLock(byte[], long, TimeUnit)}
+     */
+    private void distributedLockTest(RheaKVStore store) throws InterruptedException {
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        byte[] lockKey = makeKey("lock_test");
+        checkRegion(store, lockKey, 2);
+        final DistributedLock lock = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+        assertNotNull(lock);
+        assertTrue(lock.tryLock()); // can lock
+        assertTrue(lock.tryLock()); // reentrant lock
+        final CountDownLatch latch1 = new CountDownLatch(1);
+        Thread t = new Thread(() -> {
+            final DistributedLock lock1 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                assertTrue(!lock1.tryLock()); // fail to lock
+            } finally {
+                latch1.countDown();
+            }
+        }, "locker1-thread");
+        t.setDaemon(true);
+        t.start();
+        latch1.await();
+        final CountDownLatch latch2 = new CountDownLatch(1);
+        t = new Thread(() -> {
+            final DistributedLock lock2 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                // Waiting for the lock to expire and lock successfully
+                assertTrue(lock2.tryLock(3100, TimeUnit.MILLISECONDS));
+            } finally {
+                lock2.unlock();
+                latch2.countDown();
+            }
+        }, "locker2-thread");
+        t.setDaemon(true);
+        t.start();
+        latch2.await();
+
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock());
+        lock.unlock();
+        final CountDownLatch latch3 = new CountDownLatch(1);
+        t = new Thread(() -> {
+            final DistributedLock lock3 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                // The previous lock was not released and the lock failed.
+                assertTrue(!lock3.tryLock());
+            } finally {
+                latch3.countDown();
+            }
+        }, "locker3-thread");
+        t.setDaemon(true);
+        t.start();
+        latch3.await();
+
+        lock.unlock();
+        lock.unlock();
+        final CountDownLatch latch4 = new CountDownLatch(1);
+        t = new Thread(() -> {
+            final DistributedLock lock4 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                assertTrue(lock4.tryLock()); // lock success
+            } finally {
+                lock4.unlock();
+                latch4.countDown();
+            }
+        }, "locker4-thread");
+        t.setDaemon(true);
+        t.start();
+        latch4.await();
+
+        // lock with lease scheduler
+        final ScheduledExecutorService watchdog = Executors.newScheduledThreadPool(1);
+        final DistributedLock lockWithScheduler = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS,
+                watchdog);
+        lockWithScheduler.tryLock();
+        Thread.sleep(5000);
+
+        final CountDownLatch latch5 = new CountDownLatch(1);
+        t = new Thread(() -> {
+            final DistributedLock lock5 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                // Locking failed because lockWithScheduler is renewing
+                assertTrue(!lock5.tryLock());
+            } finally {
+                latch5.countDown();
+            }
+        }, "locker5-thread");
+        t.setDaemon(true);
+        t.start();
+        latch5.await();
+
+        lockWithScheduler.unlock();
+
+        final CountDownLatch latch6 = new CountDownLatch(1);
+        t = new Thread(() -> {
+            final DistributedLock lock6 = store.getDistributedLock(lockKey, 3, TimeUnit.SECONDS);
+            try {
+                assertTrue(lock6.tryLock()); // lock success
+            } finally {
+                lock6.unlock();
+                latch6.countDown();
+            }
+        }, "locker6-thread");
+        t.setDaemon(true);
+        t.start();
+        latch6.await();
+    }
+
+    @Test
+    public void distributedLockByLeaderTest() throws InterruptedException {
+        distributedLockTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void distributedLockByFollowerTest() throws InterruptedException {
+        distributedLockTest(getRandomFollowerStore());
+    }
+
+    private void batchWriteTest(RheaKVStore store) throws ExecutionException, InterruptedException {
+        // regions: 1 -> [null, g), 2 -> [g, t), 3 -> [t, null)
+        List> futures = Lists.newArrayList();
+        for (int i = 0; i < 100; i++) {
+            CompletableFuture f = store.put(makeKey("batch" + i), makeValue("batch"));
+            futures.add(f);
+        }
+        FutureGroup futureGroup = new FutureGroup<>(futures);
+        CompletableFuture.allOf(futureGroup.toArray()).get();
+
+        futures.clear();
+        for (int i = 0; i < 100; i++) {
+            CompletableFuture f = store.delete(makeKey("batch" + i));
+            futures.add(f);
+        }
+        futureGroup = new FutureGroup<>(futures);
+        CompletableFuture.allOf(futureGroup.toArray()).get();
+
+        for (int i = 0; i < 100; i++) {
+            byte[] value = store.bGet(makeKey("batch" + i));
+            assertNull(value);
+        }
+    }
+
+    @Test
+    public void batchWriteByLeaderTest() throws ExecutionException, InterruptedException {
+        batchWriteTest(getRandomLeaderStore());
+    }
+
+    @Test
+    public void batchWriteByFollowerTest() throws ExecutionException, InterruptedException {
+        batchWriteTest(getRandomFollowerStore());
+    }
+
+    @Test
+    public void rangeSplitTest() {
+        final RheaKVStore store = getRandomLeaderStore();
+        final long regionId = 1;
+        for (int i = 0; i < 20; i++) {
+            store.bPut("a" + i, BytesUtil.writeUtf8("split"));
+        }
+        final CliOptions opts = new CliOptions();
+        opts.setTimeoutMs(30000);
+        final RheaKVCliService cliService = RheaKVServiceFactory.createAndInitRheaKVCliService(opts);
+        final long newRegionId = 101;
+        final String groupId = JRaftHelper.getJRaftGroupId("rhea_test", regionId);
+        final Configuration conf = JRaftUtils.getConfiguration("127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183");
+        final Status st = cliService.rangeSplit(regionId, newRegionId, groupId, conf);
+        System.err.println("Status:" + st);
+        assertTrue(st.isOk());
+        final RheaKVStore newStore = getLeaderStore(101);
+        newStore.bPut("f_first_key", BytesUtil.writeUtf8("split_ok"));
+        assertArrayEquals(BytesUtil.writeUtf8("split_ok"), newStore.bGet("f_first_key"));
+    }
+
+    @Test
+    public void restartAllWithLeaderTest() throws Exception {
+        RheaKVStore store = getRandomLeaderStore();
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        store.bPut("a_get_test", makeValue("a_get_test_value"));
+        store.bPut("h_get_test", makeValue("h_get_test_value"));
+        store.bPut("z_get_test", makeValue("z_get_test_value"));
+
+        store.bGetSequence("a_seqTest", 10);
+        store.bGetSequence("h_seqTest", 11);
+        store.bGetSequence("z_seqTest", 12);
+
+        shutdown(false);
+
+        start(getStorageType(), false);
+
+        store = getRandomLeaderStore();
+
+        assertArrayEquals(makeValue("a_get_test_value"), store.bGet("a_get_test"));
+        assertArrayEquals(makeValue("h_get_test_value"), store.bGet("h_get_test"));
+        assertArrayEquals(makeValue("z_get_test_value"), store.bGet("z_get_test"));
+
+        assertEquals(10, store.bGetSequence("a_seqTest", 1).getStartValue());
+        assertEquals(11, store.bGetSequence("h_seqTest", 1).getStartValue());
+        assertEquals(12, store.bGetSequence("z_seqTest", 1).getStartValue());
+    }
+
+    @Test
+    public void restartAllWithFollowerTest() throws Exception {
+        RheaKVStore store = getRandomFollowerStore();
+        // regions: 1 -> [null, g), 2 -> [g, null)
+        store.bPut("a_get_test", makeValue("a_get_test_value"));
+        store.bPut("h_get_test", makeValue("h_get_test_value"));
+        store.bPut("z_get_test", makeValue("z_get_test_value"));
+
+        store.bGetSequence("a_seqTest", 10);
+        store.bGetSequence("h_seqTest", 11);
+        store.bGetSequence("z_seqTest", 12);
+
+        shutdown(false);
+
+        start(getStorageType(), false);
+
+        store = getRandomFollowerStore();
+
+        assertArrayEquals(makeValue("a_get_test_value"), store.bGet("a_get_test"));
+        assertArrayEquals(makeValue("h_get_test_value"), store.bGet("h_get_test"));
+        assertArrayEquals(makeValue("z_get_test_value"), store.bGet("z_get_test"));
+
+        assertEquals(10, store.bGetSequence("a_seqTest", 1).getStartValue());
+        assertEquals(11, store.bGetSequence("h_seqTest", 1).getStartValue());
+        assertEquals(12, store.bGetSequence("z_seqTest", 1).getStartValue());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockMemoryDBTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockMemoryDBTest.java
new file mode 100644
index 0000000..16f627f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockMemoryDBTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DistributedLockMemoryDBTest extends AbstractDistributedLockTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.Memory;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockRocksDBTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockRocksDBTest.java
new file mode 100644
index 0000000..b929ba6
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/DistributedLockRocksDBTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DistributedLockRocksDBTest extends AbstractDistributedLockTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithMemoryDBTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithMemoryDBTest.java
new file mode 100644
index 0000000..7dd2146
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithMemoryDBTest.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RheaKVStoreWithMemoryDBTest extends AbstractRheaKVStoreTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.Memory;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithRocksDBTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithRocksDBTest.java
new file mode 100644
index 0000000..f913a0b
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVStoreWithRocksDBTest.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVStoreWithRocksDBTest extends AbstractRheaKVStoreTest {
+
+    @Override
+    public StorageType getStorageType() {
+        return StorageType.RocksDB;
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVTestCluster.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVTestCluster.java
new file mode 100644
index 0000000..ba21eb0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/RheaKVTestCluster.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.errors.NotLeaderException;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.storage.StorageType;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class RheaKVTestCluster {
+
+    private static final Logger   LOG            = LoggerFactory.getLogger(RheaKVTestCluster.class);
+
+    public static String          DB_PATH        = "rhea_db";
+    public static String          RAFT_DATA_PATH = "rhea_raft";
+    public static Long[]          REGION_IDS     = new Long[] { 1L, 2L };
+
+    private static final String[] CONF           = { "/conf/rhea_test_cluster_1.yaml",
+            "/conf/rhea_test_cluster_2.yaml", "/conf/rhea_test_cluster_3.yaml" };
+
+    private List     stores         = new CopyOnWriteArrayList<>();
+
+    protected void start(final StorageType storageType) throws Exception {
+        start(storageType, true);
+    }
+
+    protected void start(final StorageType storageType, final boolean deleteFiles) throws Exception {
+        if (deleteFiles) {
+            deleteFiles();
+        }
+        for (final String c : CONF) {
+            final RheaKVStoreOptions opts = readOpts(c);
+            opts.getStoreEngineOptions().setStorageType(storageType);
+            final RheaKVStore rheaKVStore = new DefaultRheaKVStore();
+            if (rheaKVStore.init(opts)) {
+                stores.add(rheaKVStore);
+            } else {
+                throw new RuntimeException("Fail to init rhea kv store witch conf: " + c);
+            }
+        }
+        for (final Long id : REGION_IDS) {
+            getLeaderStore(id);
+        }
+    }
+
+    protected void shutdown() throws Exception {
+        shutdown(true);
+    }
+
+    protected void shutdown(final boolean deleteFiles) throws Exception {
+        for (final RheaKVStore store : stores) {
+            store.shutdown();
+        }
+        stores.clear();
+        if (deleteFiles) {
+            deleteFiles();
+        }
+        LOG.info("RheaKVTestCluster shutdown complete");
+    }
+
+    private RheaKVStoreOptions readOpts(final String conf) throws IOException {
+        final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        try (final InputStream in = RheaKVTestCluster.class.getResourceAsStream(conf)) {
+            return mapper.readValue(in, RheaKVStoreOptions.class);
+        }
+    }
+
+    protected RheaKVStore getRandomLeaderStore() {
+        return getLeaderStore(getRandomRegionId());
+    }
+
+    protected RheaKVStore getLeaderStore(final long regionId) {
+        for (int i = 0; i < 100; i++) {
+            for (final RheaKVStore store : stores) {
+                if (((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    return store;
+                }
+            }
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no leader on region: " + regionId);
+    }
+
+    protected RheaKVStore getRandomFollowerStore() {
+        return getFollowerStore(getRandomRegionId());
+    }
+
+    protected RheaKVStore getFollowerStore(final long regionId) {
+        for (int i = 0; i < 100; i++) {
+            final List tmp = Lists.newArrayList();
+            for (final RheaKVStore store : stores) {
+                if (!((DefaultRheaKVStore) store).isLeader(regionId)) {
+                    // maybe a learner
+                    tmp.add(store);
+                }
+            }
+            if (!tmp.isEmpty()) {
+                return tmp.get(ThreadLocalRandom.current().nextInt(tmp.size()));
+            }
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no follower/learner");
+    }
+
+    protected Long getRandomRegionId() {
+        return REGION_IDS[ThreadLocalRandom.current().nextInt(REGION_IDS.length)];
+    }
+
+    private void deleteFiles() {
+        final File dbFile = new File(DB_PATH);
+        if (dbFile.exists()) {
+            try {
+                FileUtils.forceDelete(dbFile);
+                LOG.info("delete db file: {}", dbFile.getAbsolutePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+
+        final File raftFile = new File(RAFT_DATA_PATH);
+        if (raftFile.exists()) {
+            try {
+                FileUtils.forceDelete(raftFile);
+                LOG.info("remove raft data: {}", raftFile.getAbsolutePath());
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/YamlTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/YamlTest.java
new file mode 100644
index 0000000..ff9e2ef
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rhea/YamlTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rhea;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class YamlTest {
+
+    @Test
+    public void parseStoreEngineOptionsTest() throws IOException {
+        final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        final InputStream in = YamlTest.class.getResourceAsStream("/conf/rhea_test_cluster_1.yaml");
+        final RheaKVStoreOptions opts = mapper.readValue(in, RheaKVStoreOptions.class);
+        System.out.println(opts);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/BaseKVStoreTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/BaseKVStoreTest.java
new file mode 100644
index 0000000..fe5b4e7
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/BaseKVStoreTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rocksdb;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+
+public class BaseKVStoreTest {
+
+    protected String          tempPath;
+    protected RocksRawKVStore kvStore;
+    protected RocksDBOptions  dbOptions;
+
+    protected void setup() throws Exception {
+        File file = getTempDir();
+        this.tempPath = file.getAbsolutePath();
+        this.kvStore = new RocksRawKVStore();
+        this.dbOptions = new RocksDBOptions();
+        this.dbOptions.setStatisticsCallbackIntervalSeconds(10);
+        this.dbOptions.setDbPath(this.tempPath);
+        this.kvStore.init(this.dbOptions);
+    }
+
+    protected File getTempDir() throws IOException {
+        final File file = File.createTempFile("RawRocksDBTest", "test");
+        FileUtils.forceDelete(file);
+        if (file.mkdirs()) {
+            return file;
+        }
+        throw new RuntimeException("fail to make dirs");
+    }
+
+    protected void tearDown() throws Exception {
+        this.kvStore.shutdown();
+        FileUtils.forceDelete(new File(this.tempPath));
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/RocksKVStoreTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/RocksKVStoreTest.java
new file mode 100644
index 0000000..1f6fa78
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/rocksdb/RocksKVStoreTest.java
@@ -0,0 +1,995 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.rocksdb;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.LocalFileMetaOutter.LocalFileMeta;
+import com.alipay.sofa.jraft.rhea.StoreEngineHelper;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.options.RocksDBOptions;
+import com.alipay.sofa.jraft.rhea.storage.BaseKVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.KVIterator;
+import com.alipay.sofa.jraft.rhea.storage.KVOperation;
+import com.alipay.sofa.jraft.rhea.storage.KVState;
+import com.alipay.sofa.jraft.rhea.storage.KVStateOutputList;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreAccessHelper;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreClosure;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFile;
+import com.alipay.sofa.jraft.rhea.storage.KVStoreSnapshotFileFactory;
+import com.alipay.sofa.jraft.rhea.storage.LocalLock;
+import com.alipay.sofa.jraft.rhea.storage.RawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.RocksRawKVStore;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.storage.SstColumnFamily;
+import com.alipay.sofa.jraft.rhea.storage.SyncKVStore;
+import com.alipay.sofa.jraft.rhea.storage.TestClosure;
+import com.alipay.sofa.jraft.rhea.storage.TestSnapshotReader;
+import com.alipay.sofa.jraft.rhea.storage.TestSnapshotWriter;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.concurrent.DistributedLock;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeKey;
+import static com.alipay.sofa.jraft.rhea.KeyValueTool.makeValue;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * RocksDB unit test, covering all interfaces in {@link RocksRawKVStore}.
+ *
+ * @author jiachun.fjc
+ */
+public class RocksKVStoreTest extends BaseKVStoreTest {
+
+    private static final String SNAPSHOT_ARCHIVE = "kv.zip";
+
+    @Override
+    @Before
+    public void setup() throws Exception {
+        super.setup();
+    }
+
+    @Override
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#get(byte[], KVStoreClosure)}
+     */
+    @Test
+    public void getTest() {
+        final byte[] key = makeKey("get_test");
+        byte[] value = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertNull(value);
+
+        value = makeValue("get_test_value");
+        this.kvStore.put(key, value, null);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(value, newValue);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#multiGet(List, KVStoreClosure)}
+     */
+    @Test
+    public void multiGetTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("multi_test_key_" + i);
+            byte[] value = makeValue("multi_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        Map mapResult = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.multiGet(keyList, closure);
+            }
+        }.apply(this.kvStore);
+        for (int i = 0; i < keyList.size(); i++) {
+            byte[] key = keyList.get(i);
+            assertArrayEquals(mapResult.get(ByteArray.wrap(key)), valueList.get(i));
+        }
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#localIterator()}
+     */
+    @Test
+    public void getLocalIteratorTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("iterator_test_key_" + i);
+            byte[] value = makeValue("iterator_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+
+        final List entries = Lists.newArrayList();
+        try (final KVIterator it = this.kvStore.localIterator()) {
+            it.seekToFirst();
+            while (it.isValid()) {
+                entries.add(new KVEntry(it.key(), it.value()));
+                it.next();
+            }
+        } catch (final Exception e) {
+            e.printStackTrace();
+        }
+
+        assertEquals(entries.size(), keyList.size());
+
+        for (int i = 0; i < keyList.size(); i++) {
+            assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+        }
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#containsKey(byte[], KVStoreClosure)}
+     */
+    @Test
+    public void containsKeyTest() {
+        final byte[] key = makeKey("contains_key_test");
+        Boolean isContains = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.containsKey(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertFalse(isContains);
+
+        final byte[] value = makeValue("contains_key_test_value");
+        this.kvStore.put(key, value, null);
+        isContains = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.containsKey(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertTrue(isContains);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#scan(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void scanTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            byte[] value = makeValue("scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            byte[] value = makeValue("no_scan_test_value_" + i);
+            this.kvStore.put(key, value, null);
+        }
+        List entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(makeKey("scan_test_key_"), makeKey("scan_test_key_" + 99), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), keyList.size());
+        for (int i = 0; i < keyList.size(); i++) {
+            assertArrayEquals(keyList.get(i), entries.get(i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(i).getValue());
+        }
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(null, null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#reverseScan(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void reverseScanTest() {
+        final List keyList = Lists.newArrayList();
+        final List valueList = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("scan_test_key_" + i);
+            byte[] value = makeValue("scan_test_value_" + i);
+            keyList.add(key);
+            valueList.add(value);
+            this.kvStore.put(key, value, null);
+        }
+        for (int i = 0; i < 10; i++) {
+            byte[] key = makeKey("no_scan_test_key_" + i);
+            byte[] value = makeValue("no_scan_test_value_" + i);
+            this.kvStore.put(key, value, null);
+        }
+
+        List entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(makeKey("scan_test_key_" + 99), makeKey("scan_test_key_"), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), keyList.size());
+        for (int i = keyList.size() - 1; i >= 0; i--) {
+            assertArrayEquals(keyList.get(i), entries.get(keyList.size() - 1 - i).getKey());
+            assertArrayEquals(valueList.get(i), entries.get(keyList.size() - 1 - i).getValue());
+        }
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(makeKey("scan_test_key_" + 99), null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+
+        entries = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.reverseScan(null, null, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), 20);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#getSequence(byte[], int, KVStoreClosure)}
+     */
+    @Test
+    public void getSequenceTest() throws InterruptedException {
+        final byte[] seqKey = makeKey("seq_test");
+        Sequence sequence = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 199, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence.getStartValue(), 0);
+        assertEquals(sequence.getEndValue(), 199);
+
+        Sequence sequence2 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 10, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence2.getStartValue(), 199);
+        assertEquals(sequence2.getEndValue(), 209);
+        this.kvStore.resetSequence(seqKey, null);
+        Sequence sequence3 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 11, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence3.getStartValue(), 0);
+        assertEquals(sequence3.getEndValue(), 11);
+
+        // read-only
+        Sequence sequence4 = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(seqKey, 0, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(sequence4.getStartValue(), 11);
+        assertEquals(sequence4.getEndValue(), 11);
+
+        KVStoreClosure assertFailed = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals("Fail to [GET_SEQUENCE], step must >= 0", status.getErrorMsg());
+            }
+        };
+        this.kvStore.getSequence(seqKey, -1, assertFailed);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#getSafeEndValueForSequence(long, int)}
+     */
+    @Test
+    public void getSafeEndValueForSequenceTest() {
+        long startVal = 1;
+        assertEquals(2, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        startVal = Long.MAX_VALUE - 1;
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 2));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, Integer.MAX_VALUE));
+        startVal = Long.MAX_VALUE;
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 0));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 1));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, 2));
+        assertEquals(Long.MAX_VALUE, this.kvStore.getSafeEndValueForSequence(startVal, Integer.MAX_VALUE));
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#put(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void putTest() {
+        final byte[] key = makeKey("put_test");
+        TestClosure closure = new TestClosure();
+        this.kvStore.get(key, closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        this.kvStore.put(key, value, null);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(value, newValue);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#getAndPut(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void getAndPutTest() {
+        final byte[] key = makeKey("put_test");
+        TestClosure closure = new TestClosure();
+        this.kvStore.get(key, closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+
+        value = makeValue("put_test_value");
+        KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals(status, Status.OK());
+            }
+        };
+        this.kvStore.getAndPut(key, value, kvStoreClosure);
+        assertNull(kvStoreClosure.getData());
+
+        byte[] newVal = makeValue("put_test_value_new");
+        this.kvStore.getAndPut(key, newVal, kvStoreClosure);
+        assertArrayEquals(value, (byte[]) kvStoreClosure.getData());
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#compareAndPut(byte[], byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void compareAndPutTest() {
+        final byte[] key = makeKey("put_test");
+        final byte[] value = makeValue("put_test_value");
+        this.kvStore.put(key, value, null);
+
+        final byte[] update = makeValue("put_test_update");
+        KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+            @Override
+            public void run(Status status) {
+                assertEquals(status, Status.OK());
+            }
+        };
+        this.kvStore.compareAndPut(key, value, update, kvStoreClosure);
+        assertEquals(kvStoreClosure.getData(), Boolean.TRUE);
+        byte[] newValue = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(update, newValue);
+
+        this.kvStore.compareAndPut(key, value, update, kvStoreClosure);
+        assertEquals(kvStoreClosure.getData(), Boolean.FALSE);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#batchCompareAndPut(KVStateOutputList)}
+     */
+    @Test
+    public void batchCompareAndPutTest() {
+        final KVStateOutputList kvStates = KVStateOutputList.newInstance();
+        final int batchWriteSize = RocksRawKVStore.MAX_BATCH_WRITE_SIZE + 1;
+        for (int i = 1; i <= batchWriteSize; i++) {
+            final byte[] key = makeKey("put_test" + i);
+            final byte[] value = makeValue("put_test_value" + i);
+            kvStates.add(KVState.of(KVOperation.createPut(key, value), null));
+        }
+        this.kvStore.batchPut(kvStates);
+        kvStates.clear();
+
+        for (int i = 1; i <= batchWriteSize; i++) {
+            final byte[] key = makeKey("put_test" + i);
+            final byte[] value = makeValue("put_test_value" + i);
+            final byte[] update = makeValue("put_test_update" + i);
+            KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+                @Override
+                public void run(Status status) {
+                    assertEquals(status, Status.OK());
+                }
+            };
+            kvStates.add(KVState.of(KVOperation.createCompareAndPut(key, update, value), kvStoreClosure));
+        }
+        this.kvStore.batchCompareAndPut(kvStates);
+        kvStates.forEach(kvState -> assertEquals(kvState.getDone().getData(), Boolean.FALSE));
+        kvStates.clear();
+
+        for (int i = 1; i <= batchWriteSize; i++) {
+            final byte[] key = makeKey("put_test" + i);
+            final byte[] value = makeValue("put_test_value" + i);
+            final byte[] update = makeValue("put_test_update" + i);
+            KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+                @Override
+                public void run(Status status) {
+                    assertEquals(status, Status.OK());
+                }
+            };
+            kvStates.add(KVState.of(KVOperation.createCompareAndPut(key, value, update), kvStoreClosure));
+        }
+        this.kvStore.batchCompareAndPut(kvStates);
+        kvStates.forEach(kvState -> assertEquals(kvState.getDone().getData(), Boolean.TRUE));
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#merge(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void mergeTest() {
+        final byte[] key = BytesUtil.writeUtf8("merge_test");
+        final byte[] bytes1 = BytesUtil.writeUtf8("a");
+        final byte[] bytes2 = BytesUtil.writeUtf8("b");
+        final byte[] bytes3 = BytesUtil.writeUtf8("c");
+        final byte[] bytes4 = BytesUtil.writeUtf8("d");
+        this.kvStore.put(key, bytes1, null);
+        this.kvStore.merge(key, bytes2, null);
+        this.kvStore.merge(key, bytes3, null);
+        this.kvStore.merge(key, bytes4, null);
+
+        final byte[] val = new SyncKVStore() {
+
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+        assertArrayEquals(BytesUtil.writeUtf8("a,b,c,d"), val);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#put(List, KVStoreClosure)}
+     */
+    @Test
+    public void putListTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("batch_put_test_key" + i), makeValue("batch_put_test_value" + i)));
+        }
+        this.kvStore.put(entries, null);
+        final List entries2 = new SyncKVStore>() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.scan(makeKey("batch_put_test_key"), makeKey("batch_put_test_key" + 99), closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(entries.size(), entries2.size());
+        for (int i = 0; i < entries.size(); i++) {
+            assertArrayEquals(entries.get(i).getKey(), entries2.get(i).getKey());
+            assertArrayEquals(entries.get(i).getValue(), entries2.get(i).getValue());
+        }
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#putIfAbsent(byte[], byte[], KVStoreClosure)}
+     */
+    @Test
+    public void putIfAbsentTest() {
+        byte[] key = makeKey("put_if_absent_test");
+        byte[] value = makeValue("put_if_absent_test_value");
+        TestClosure closure = new TestClosure();
+        this.kvStore.putIfAbsent(key, value, closure);
+        assertNull(closure.getData());
+        this.kvStore.putIfAbsent(key, value, closure);
+        assertArrayEquals(value, (byte[]) closure.getData());
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#batchPutIfAbsent(KVStateOutputList)}
+     */
+    @Test
+    public void batchPutIfAbsentTest() {
+        final KVStateOutputList kvStates = KVStateOutputList.newInstance();
+        final int batchWriteSize = RocksRawKVStore.MAX_BATCH_WRITE_SIZE + 1;
+        for (int i = 1; i <= batchWriteSize; i++) {
+            final byte[] key = makeKey("put_test" + i);
+            final byte[] value = makeValue("put_test_value" + i);
+            KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+                @Override
+                public void run(Status status) {
+                    assertEquals(status, Status.OK());
+                }
+            };
+            kvStates.add(KVState.of(KVOperation.createPutIfAbsent(key, value), kvStoreClosure));
+        }
+        this.kvStore.batchPutIfAbsent(kvStates);
+        kvStates.forEach(kvState -> assertNull(kvState.getDone().getData()));
+        kvStates.clear();
+
+        for (int i = 1; i <= batchWriteSize; i++) {
+            final byte[] key = makeKey("put_test" + i);
+            final byte[] value = makeValue("put_test_value" + i);
+            KVStoreClosure kvStoreClosure = new BaseKVStoreClosure() {
+                @Override
+                public void run(Status status) {
+                    assertEquals(status, Status.OK());
+                }
+            };
+            kvStates.add(KVState.of(KVOperation.createPutIfAbsent(key, value), kvStoreClosure));
+        }
+        this.kvStore.batchPutIfAbsent(kvStates);
+        kvStates.forEach(kvState -> assertArrayEquals(kvState.getOp().getValue(), (byte[]) kvState.getDone().getData()));
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#tryLockWith(byte[], byte[], boolean, DistributedLock.Acquirer, KVStoreClosure)}
+     */
+    @Test
+    public void tryLockWith() throws InterruptedException {
+        byte[] lockKey = makeKey("lock_test");
+        final DistributedLock lock = new LocalLock(lockKey, 3, TimeUnit.SECONDS, this.kvStore);
+        assertNotNull(lock);
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock());
+        assertTrue(lock.tryLock(3001, TimeUnit.MILLISECONDS));
+        final CountDownLatch latch = new CountDownLatch(1);
+        new Thread(() -> {
+            final DistributedLock lock2 = new LocalLock(lockKey, 3, TimeUnit.SECONDS, this.kvStore);
+            try {
+                assertFalse(lock2.tryLock());
+            } finally {
+                latch.countDown();
+            }
+        }, "lock1-thread").start();
+        latch.await();
+        lock.unlock();
+        assertTrue(lock.tryLock());
+        lock.unlock();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("del_test" + i), makeValue("del_test_value")));
+        }
+        this.kvStore.put(entries, null);
+        this.kvStore.delete(makeKey("del_test5"), null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("del_test"), makeKey("del_test" + 99), closure);
+        List entries2 = (List) closure.getData();
+        assertEquals(entries.size() - 1, entries2.size());
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_test5"), closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteRangeTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            entries.add(new KVEntry(makeKey("del_range_test" + i), makeValue("del_range_test" + i)));
+        }
+        this.kvStore.put(entries, null);
+        // delete [del_range_test5, del_range_test8)
+        this.kvStore.deleteRange(makeKey("del_range_test5"), makeKey("del_range_test8"), null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("del_range_test"), makeKey("del_range_test" + 99), closure);
+        final List entries2 = (List) closure.getData();
+        assertEquals(entries.size() - 3, entries2.size());
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test5"), closure);
+        byte[] value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test6"), closure);
+        value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test7"), closure);
+        value = (byte[]) closure.getData();
+        assertNull(value);
+        closure = new TestClosure();
+        this.kvStore.get(makeKey("del_range_test8"), closure);
+        value = (byte[]) closure.getData();
+        assertNotNull(value);
+    }
+
+    /**
+     * Test method: {@link RocksRawKVStore#delete(List, KVStoreClosure)}
+     */
+    @SuppressWarnings("unchecked")
+    @Test
+    public void deleteListTest() {
+        final List entries = Lists.newArrayList();
+        final List keys = Lists.newArrayList();
+        for (int i = 0; i < 10; i++) {
+            final byte[] key = makeKey("batch_del_test" + i);
+            entries.add(new KVEntry(key, makeValue("batch_del_test_value")));
+            keys.add(key);
+        }
+        this.kvStore.put(entries, null);
+        this.kvStore.delete(keys, null);
+        TestClosure closure = new TestClosure();
+        this.kvStore.scan(makeKey("batch_del_test"), makeKey("batch_del_test" + 99), closure);
+        List entries2 = (List) closure.getData();
+        assertEquals(0, entries2.size());
+        for (int i = 0; i < keys.size(); i++) {
+            closure = new TestClosure();
+            this.kvStore.get(keys.get(i), closure);
+            byte[] value = (byte[]) closure.getData();
+            assertNull(value);
+        }
+    }
+
+    @Test
+    public void slowSnapshotTest() throws Exception {
+        this.dbOptions.setFastSnapshot(false);
+        snapshotTest();
+    }
+
+    @Test
+    public void fastSnapshotTest() throws Exception {
+        this.dbOptions.setFastSnapshot(true);
+        snapshotTest();
+    }
+
+    public void snapshotTest() throws Exception {
+        final File backupDir = new File("backup");
+        if (backupDir.exists()) {
+            FileUtils.deleteDirectory(backupDir);
+        }
+        FileUtils.forceMkdir(backupDir);
+        for (int i = 0; i < 100000; i++) {
+            final String v = String.valueOf(i);
+            this.kvStore.put(makeKey(v), makeValue(v), null);
+        }
+
+        final Region region = new Region();
+        KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 2);
+        final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(backupDir.getAbsolutePath());
+        final CountDownLatch latch = new CountDownLatch(1);
+        final Closure done = status -> {
+            assertTrue(status.isOk());
+            latch.countDown();
+        };
+        kvStoreSnapshotFile.save(snapshotWriter, region, done, snapshotPool);
+        latch.await();
+        final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
+        assertNotNull(meta);
+
+        assertNotNull(get(makeKey("1")));
+
+        this.kvStore.put(makeKey("100001"), makeValue("100001"), null);
+        assertNotNull(get(makeKey("100001")));
+
+        this.kvStore.shutdown();
+        FileUtils.deleteDirectory(new File(this.tempPath));
+        FileUtils.forceMkdir(new File(this.tempPath));
+        this.kvStore = new RocksRawKVStore();
+        this.kvStore.init(this.dbOptions);
+
+        assertNull(get(makeKey("1")));
+
+        final TestSnapshotReader snapshotReader = new TestSnapshotReader(snapshotWriter.metaTable, backupDir.getAbsolutePath());
+        kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final boolean ret = kvStoreSnapshotFile.load(snapshotReader, region);
+        assertTrue(ret);
+
+        for (int i = 0; i < 100000; i++) {
+            final String v = String.valueOf(i);
+            assertArrayEquals(makeValue(v), get(makeKey(v)));
+        }
+
+        // key[100001] is put after the snapshot, so key[100001] should not exist.
+        assertNull(get(makeKey("100001")));
+
+        FileUtils.deleteDirectory(backupDir);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
+    }
+
+    private byte[] get(final byte[] key) {
+        return new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.get(key, closure);
+            }
+        }.apply(this.kvStore);
+    }
+
+    @Test
+    public void syncMultiGroupSnapshotTest() throws Exception {
+        this.dbOptions.setAsyncSnapshot(false);
+        multiGroupSnapshotTest();
+    }
+
+    @Test
+    public void asyncMultiGroupSnapshotTest() throws Exception {
+        this.dbOptions.setAsyncSnapshot(true);
+        multiGroupSnapshotTest();
+    }
+
+    public void multiGroupSnapshotTest() throws Exception {
+        final File backupDir = new File("multi-backup");
+        if (backupDir.exists()) {
+            FileUtils.deleteDirectory(backupDir);
+        }
+
+        final List regions = Lists.newArrayList();
+        regions.add(new Region(1, makeKey("0"), makeKey("1"), null, null));
+        regions.add(new Region(2, makeKey("1"), makeKey("2"), null, null));
+        regions.add(new Region(3, makeKey("2"), makeKey("3"), null, null));
+        regions.add(new Region(4, makeKey("3"), makeKey("4"), null, null));
+        regions.add(new Region(5, makeKey("4"), makeKey("5"), null, null));
+
+        for (int i = 0; i < 5; i++) {
+            final String v = String.valueOf(i);
+            this.kvStore.put(makeKey(v), makeValue(v), null);
+        }
+        for (int i = 0; i < 5; i++) {
+            this.kvStore.getSequence(makeKey(i + "_seq_test"), 10, null);
+        }
+
+        KVStoreSnapshotFile kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        final ExecutorService snapshotPool = StoreEngineHelper.createSnapshotExecutor(1, 5);
+        final List writers = Lists.newArrayList();
+        for (int i = 0; i < 4; i++) {
+            final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
+            final TestSnapshotWriter snapshotWriter = new TestSnapshotWriter(p.toString());
+            writers.add(snapshotWriter);
+            final CountDownLatch latch = new CountDownLatch(1);
+            final Closure done = status -> {
+                assertTrue(status.isOk());
+                latch.countDown();
+            };
+            kvStoreSnapshotFile.save(snapshotWriter, regions.get(i), done, snapshotPool);
+            latch.await();
+            final LocalFileMeta meta = (LocalFileMeta) snapshotWriter.getFileMeta(SNAPSHOT_ARCHIVE);
+            assertNotNull(meta);
+        }
+
+        this.kvStore.shutdown();
+        FileUtils.deleteDirectory(new File(this.tempPath));
+        FileUtils.forceMkdir(new File(this.tempPath));
+        this.kvStore = new RocksRawKVStore();
+        this.kvStore.init(this.dbOptions);
+
+        kvStoreSnapshotFile = KVStoreSnapshotFileFactory.getKVStoreSnapshotFile(this.kvStore);
+        for (int i = 0; i < 4; i++) {
+            final Path p = Paths.get(backupDir.getAbsolutePath(), String.valueOf(i));
+            final TestSnapshotReader snapshotReader = new TestSnapshotReader(writers.get(i).metaTable, p.toString());
+            final boolean ret = kvStoreSnapshotFile.load(snapshotReader, regions.get(i));
+            assertTrue(ret);
+        }
+
+        for (int i = 0; i < 4; i++) {
+            final String v = String.valueOf(i);
+            final byte[] seqKey = makeKey(i + "_seq_test");
+            assertArrayEquals(makeValue(v), get(makeKey(v)));
+            final Sequence sequence = new SyncKVStore() {
+                @Override
+                public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                    kvStore.getSequence(seqKey, 10, closure);
+                }
+            }.apply(this.kvStore);
+            assertEquals(10L, sequence.getStartValue());
+            assertEquals(20L, sequence.getEndValue());
+        }
+
+        assertNull(get(makeKey("5")));
+        final Sequence sequence = new SyncKVStore() {
+            @Override
+            public void execute(RawKVStore kvStore, KVStoreClosure closure) {
+                kvStore.getSequence(makeKey("4_seq_test"), 10, closure);
+            }
+        }.apply(this.kvStore);
+        assertEquals(0L, sequence.getStartValue());
+
+        FileUtils.deleteDirectory(backupDir);
+        ExecutorServiceHelper.shutdownAndAwaitTermination(snapshotPool);
+    }
+
+    @Test
+    public void statisticsTest() throws Exception {
+        // put
+        putTest();
+
+        // close db
+        Method closeRocksDB = this.kvStore.getClass().getDeclaredMethod("closeRocksDB");
+        closeRocksDB.setAccessible(true);
+        closeRocksDB.invoke(this.kvStore);
+        Thread.sleep(5000);
+
+        // open db
+        Method openRocksDB = this.kvStore.getClass().getDeclaredMethod("openRocksDB", RocksDBOptions.class);
+        openRocksDB.setAccessible(true);
+        openRocksDB.invoke(this.kvStore, super.dbOptions);
+
+        // get
+        getTest();
+    }
+
+    @Test
+    public void getApproximateKeysInRangeTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10000; i++) {
+            entries.add(new KVEntry(makeKey("approximate_test" + i), makeValue("approximate_test_value")));
+        }
+        this.kvStore.put(entries, null);
+
+        long approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test" + 9999), null);
+        assertEquals(1, approximateKeys);
+        approximateKeys = this.kvStore.getApproximateKeysInRange(null, makeKey("approximate_test" + 9999));
+        assertEquals(10000, approximateKeys);
+        approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test" + 9990),
+            makeKey("approximate_test" + 9999));
+        assertEquals(10, approximateKeys);
+    }
+
+    @Test
+    public void jumpOverTest() {
+        final List entries = Lists.newArrayList();
+        for (int i = 0; i < 10000; i++) {
+            entries.add(new KVEntry(makeKey("approximate_test" + i), makeValue("approximate_test_value")));
+        }
+        this.kvStore.put(entries, null);
+        final byte[] endKey = this.kvStore.jumpOver(makeKey("approximate_test0000"), 1000);
+        final long approximateKeys = this.kvStore.getApproximateKeysInRange(makeKey("approximate_test0000"), endKey);
+        assertEquals(1000, approximateKeys, 100);
+    }
+
+    @Test
+    public void initFencingTokenTest() throws Exception {
+        final Method getNextFencingMethod = RocksRawKVStore.class
+            .getDeclaredMethod("getNextFencingToken", byte[].class);
+        getNextFencingMethod.setAccessible(true);
+        final List parentKeys = Lists.newArrayList();
+        parentKeys.add(null); // startKey == null
+        parentKeys.add(BytesUtil.writeUtf8("parent"));
+        for (int i = 0; i < 2; i++) {
+            final byte[] parentKey = parentKeys.get(i);
+            final byte[] childKey = BytesUtil.writeUtf8("child");
+            assertEquals(1L, getNextFencingMethod.invoke(this.kvStore, (Object) parentKey));
+            assertEquals(2L, getNextFencingMethod.invoke(this.kvStore, (Object) parentKey));
+            this.kvStore.initFencingToken(parentKey, childKey);
+            assertEquals(3L, getNextFencingMethod.invoke(this.kvStore, (Object) childKey));
+            assertEquals(4L, getNextFencingMethod.invoke(this.kvStore, (Object) childKey));
+        }
+    }
+
+    @Test
+    public void sstFilesTest() throws IOException {
+        for (int i = 0; i < 10000; i++) {
+            byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            this.kvStore.put(bytes, bytes, null);
+        }
+        this.kvStore.getSequence(BytesUtil.writeUtf8("seq"), 100, null);
+        final File defaultSstFile = new File("default.sst");
+        final File seqSstFile = new File("seq.sst");
+        final File lockingFile = new File("locking.sst");
+        final File fencingFile = new File("fencing.sst");
+        if (defaultSstFile.exists()) {
+            FileUtils.forceDelete(defaultSstFile);
+        }
+        if (seqSstFile.exists()) {
+            FileUtils.forceDelete(seqSstFile);
+        }
+        if (lockingFile.exists()) {
+            FileUtils.forceDelete(lockingFile);
+        }
+        if (fencingFile.exists()) {
+            FileUtils.forceDelete(fencingFile);
+        }
+        final EnumMap sstFileTable = new EnumMap<>(SstColumnFamily.class);
+        sstFileTable.put(SstColumnFamily.DEFAULT, defaultSstFile);
+        sstFileTable.put(SstColumnFamily.SEQUENCE, seqSstFile);
+        sstFileTable.put(SstColumnFamily.LOCKING, lockingFile);
+        sstFileTable.put(SstColumnFamily.FENCING, fencingFile);
+        KVStoreAccessHelper.createSstFiles(this.kvStore, sstFileTable, null, null);
+        // remove keys
+        for (int i = 0; i < 10000; i++) {
+            byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            this.kvStore.delete(bytes, null);
+        }
+        this.kvStore.resetSequence(BytesUtil.writeUtf8("seq"), null);
+
+        for (int i = 0; i < 10000; i++) {
+            byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            TestClosure closure = new TestClosure();
+            this.kvStore.get(bytes, closure);
+            assertNull(closure.getData());
+        }
+        KVStoreAccessHelper.ingestSstFiles(this.kvStore, sstFileTable);
+        if (defaultSstFile.exists()) {
+            FileUtils.forceDelete(defaultSstFile);
+        }
+        if (seqSstFile.exists()) {
+            FileUtils.forceDelete(seqSstFile);
+        }
+        if (lockingFile.exists()) {
+            FileUtils.forceDelete(lockingFile);
+        }
+        if (fencingFile.exists()) {
+            FileUtils.forceDelete(fencingFile);
+        }
+        for (int i = 0; i < 10000; i++) {
+            byte[] bytes = BytesUtil.writeUtf8(String.valueOf(i));
+            TestClosure closure = new TestClosure();
+            this.kvStore.get(bytes, closure);
+            assertArrayEquals((byte[]) closure.getData(), bytes);
+        }
+        TestClosure closure = new TestClosure();
+        this.kvStore.getSequence(BytesUtil.writeUtf8("seq"), 100, closure);
+        assertEquals(((Sequence) closure.getData()).getStartValue(), 100);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategyTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategyTest.java
new file mode 100644
index 0000000..86ef222
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/JDKZipStrategyTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.util.CRC64;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.Checksum;
+
+/**
+ * @author hzh
+ */
+public class JDKZipStrategyTest {
+    private File        sourceDir;
+
+    private ZipStrategy zipStrategy;
+
+    @Before
+    public void startup() throws IOException {
+        this.sourceDir = new File("zip_test");
+        this.zipStrategy = new JDKZipStrategy();
+
+        if (this.sourceDir.exists()) {
+            FileUtils.forceDelete(this.sourceDir);
+        }
+        FileUtils.forceMkdir(this.sourceDir);
+        final File f1 = Paths.get(this.sourceDir.getAbsolutePath(), "f1").toFile();
+        FileUtils.write(f1, "f1");
+        final File d1 = Paths.get(this.sourceDir.getAbsolutePath(), "d1").toFile();
+        FileUtils.forceMkdir(d1);
+        final File f11 = Paths.get(d1.getAbsolutePath(), "f11").toFile();
+        FileUtils.write(f11, "f11");
+
+        final File d2 = Paths.get(d1.getAbsolutePath(), "d2").toFile();
+        FileUtils.forceMkdir(d2);
+
+        final File d3 = Paths.get(d2.getAbsolutePath(), "d3").toFile();
+        FileUtils.forceMkdir(d3);
+
+        final File f31 = Paths.get(d3.getAbsolutePath(), "f31").toFile();
+        FileUtils.write(f31, "f32");
+
+    }
+
+    @After
+    public void teardown() throws IOException {
+        FileUtils.forceDelete(this.sourceDir);
+    }
+
+    @Test
+    public void zipTest() throws Throwable {
+        final String rootPath = this.sourceDir.toPath().toAbsolutePath().getParent().toString();
+        final Path outPath = Paths.get(rootPath, "kv.zip");
+        final Checksum c1 = new CRC64();
+        zipStrategy.compress(rootPath, "zip_test", outPath.toString(), c1);
+
+        System.out.println(Long.toHexString(c1.getValue()));
+
+        final Checksum c2 = new CRC64();
+        zipStrategy.deCompress(Paths.get(rootPath, "kv.zip").toString(), rootPath, c2);
+
+        Assert.assertEquals(c1.getValue(), c2.getValue());
+
+        FileUtils.forceDelete(outPath.toFile());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategyTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategyTest.java
new file mode 100644
index 0000000..081c544
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ParallelZipStrategyTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.util.CRC64;
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.Checksum;
+
+/**
+ * @author hzh
+ */
+public class ParallelZipStrategyTest {
+    private File        sourceDir;
+    private ZipStrategy zipStrategy;
+
+    @Before
+    public void startup() throws IOException {
+        this.sourceDir = new File("zip_test");
+        this.zipStrategy = new ParallelZipStrategy(9, 6);
+
+        if (this.sourceDir.exists()) {
+            FileUtils.forceDelete(this.sourceDir);
+        }
+        FileUtils.forceMkdir(this.sourceDir);
+        final File f1 = Paths.get(this.sourceDir.getAbsolutePath(), "f1").toFile();
+        FileUtils.write(f1, "f1");
+        final File d1 = Paths.get(this.sourceDir.getAbsolutePath(), "d1").toFile();
+        FileUtils.forceMkdir(d1);
+        final File f11 = Paths.get(d1.getAbsolutePath(), "f11").toFile();
+        FileUtils.write(f11, "f11");
+
+        final File d2 = Paths.get(d1.getAbsolutePath(), "d2").toFile();
+        FileUtils.forceMkdir(d2);
+
+        final File d3 = Paths.get(d2.getAbsolutePath(), "d3").toFile();
+        FileUtils.forceMkdir(d3);
+
+        final File f31 = Paths.get(d3.getAbsolutePath(), "f31").toFile();
+        FileUtils.write(f31, "f32");
+    }
+
+    @After
+    public void teardown() throws IOException {
+        FileUtils.forceDelete(this.sourceDir);
+    }
+
+    @Test
+    public void zipTest() throws Throwable {
+        final String rootPath = this.sourceDir.toPath().toAbsolutePath().getParent().toString();
+        final Path outPath = Paths.get(rootPath, "kv.zip");
+        final Checksum c1 = new CRC64();
+        zipStrategy.compress(rootPath, "zip_test", outPath.toString(), c1);
+
+        System.out.println(Long.toHexString(c1.getValue()));
+
+        final Checksum c2 = new CRC64();
+        zipStrategy.deCompress(Paths.get(rootPath, "kv.zip").toString(), rootPath, c2);
+
+        Assert.assertEquals(c1.getValue(), c2.getValue());
+
+        FileUtils.forceDelete(outPath.toFile());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManagerTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManagerTest.java
new file mode 100644
index 0000000..3e039e4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/storage/zip/ZipStrategyManagerTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.storage.zip;
+
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import org.junit.Test;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author blake.qiu
+ */
+public class ZipStrategyManagerTest {
+
+    @Test
+    public void testInit() {
+        RheaKVStoreOptions opts = new RheaKVStoreOptions();
+        opts.setUseParallelCompress(true);
+        ZipStrategyManager.init(opts);
+        ZipStrategy zipStrategy = ZipStrategyManager.getZipStrategy(ZipStrategyManager.PARALLEL_STRATEGY);
+        assertNotNull(zipStrategy);
+        assertTrue(zipStrategy instanceof ParallelZipStrategy);
+    }
+
+}
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsBenchmark.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsBenchmark.java
new file mode 100644
index 0000000..fd7a718
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsBenchmark.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.concurrent.TimeUnit;
+
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+import org.openjdk.jmh.runner.options.TimeValue;
+
+/**
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("all")
+@State(Scope.Benchmark)
+public class VarIntsBenchmark {
+
+    /**
+    Benchmark                                     Mode     Cnt       Score      Error   Units
+    VarIntsBenchmark.int64                       thrpt       3       3.048 ±    1.002  ops/ns
+    VarIntsBenchmark.varInt64                    thrpt       3       0.931 ±    0.020  ops/ns
+    VarIntsBenchmark.int64                        avgt       3       0.319 ±    0.023   ns/op
+    VarIntsBenchmark.varInt64                     avgt       3       1.074 ±    0.080   ns/op
+    VarIntsBenchmark.int64                      sample  799721      51.160 ±    3.511   ns/op
+    VarIntsBenchmark.int64:int64·p0.00          sample               6.000              ns/op
+    VarIntsBenchmark.int64:int64·p0.50          sample              39.000              ns/op
+    VarIntsBenchmark.int64:int64·p0.90          sample              41.000              ns/op
+    VarIntsBenchmark.int64:int64·p0.95          sample              45.000              ns/op
+    VarIntsBenchmark.int64:int64·p0.99          sample              92.000              ns/op
+    VarIntsBenchmark.int64:int64·p0.999         sample             760.278              ns/op
+    VarIntsBenchmark.int64:int64·p0.9999        sample           17313.779              ns/op
+    VarIntsBenchmark.int64:int64·p1.00          sample          697344.000              ns/op
+    VarIntsBenchmark.varInt64                   sample  861436      46.786 ±    1.051   ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.00    sample               1.000              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.50    sample              39.000              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.90    sample              42.000              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.95    sample              42.000              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.99    sample              53.000              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.999   sample             301.126              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p0.9999  sample           13407.906              ns/op
+    VarIntsBenchmark.varInt64:varInt64·p1.00    sample           40448.000              ns/op
+    VarIntsBenchmark.int64                          ss       3    2254.333 ± 6750.580   ns/op
+    VarIntsBenchmark.varInt64                       ss       3    2859.333 ± 3305.354   ns/op
+     */
+
+    static long SMALL_VAL = 64;
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void varInt64() {
+        byte[] bytes = VarInts.writeVarInt64(SMALL_VAL);
+        VarInts.readVarInt64(bytes);
+    }
+
+    @Benchmark
+    @BenchmarkMode(Mode.All)
+    @OutputTimeUnit(TimeUnit.NANOSECONDS)
+    public void int64() {
+        byte[] bytes = new byte[8];
+        putLong(bytes, 0, SMALL_VAL);
+        getLong(bytes, 0);
+    }
+
+    public static long getLong(byte[] b, int off) {
+        return (b[off + 7] & 0xFFL) + ((b[off + 6] & 0xFFL) << 8) + ((b[off + 5] & 0xFFL) << 16)
+               + ((b[off + 4] & 0xFFL) << 24) + ((b[off + 3] & 0xFFL) << 32) + ((b[off + 2] & 0xFFL) << 40)
+               + ((b[off + 1] & 0xFFL) << 48) + ((long) b[off] << 56);
+    }
+
+    public static void putLong(byte[] b, int off, long val) {
+        b[off + 7] = (byte) val;
+        b[off + 6] = (byte) (val >>> 8);
+        b[off + 5] = (byte) (val >>> 16);
+        b[off + 4] = (byte) (val >>> 24);
+        b[off + 3] = (byte) (val >>> 32);
+        b[off + 2] = (byte) (val >>> 40);
+        b[off + 1] = (byte) (val >>> 48);
+        b[off] = (byte) (val >>> 56);
+    }
+
+    public static void main(String[] args) throws RunnerException {
+        Options opt = new OptionsBuilder() //
+            .include(VarIntsBenchmark.class.getSimpleName()) //
+            .warmupIterations(3) //
+            .warmupTime(TimeValue.seconds(10)) //
+            .measurementIterations(3) //
+            .measurementTime(TimeValue.seconds(10)) //
+            .forks(1) //
+            .build();
+
+        new Runner(opt).run();
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsTest.java
new file mode 100644
index 0000000..8658413
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/VarIntsTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.util.Random;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jiachun.fjc
+ */
+@SuppressWarnings("all")
+public class VarIntsTest {
+
+    @Test
+    public void varInt32Test() {
+        for (int i = 0; i < 10000; i++) {
+            readWriteVarInt32();
+        }
+    }
+
+    private void readWriteVarInt32() {
+        int value = new Random().nextInt();
+        byte[] bytes = VarInts.writeVarInt32(value);
+        int newValue = VarInts.readVarInt32(bytes);
+        assertEquals(value, newValue);
+    }
+
+    @Test
+    public void varInt64Test() {
+        for (int i = 0; i < 10000; i++) {
+            readWriteVarInt64();
+        }
+    }
+
+    private void readWriteVarInt64() {
+        long value = new Random().nextInt();
+        byte[] bytes = VarInts.writeVarInt64(value);
+        long newValue = VarInts.readVarInt64(bytes);
+        assertEquals(value, newValue);
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/ZipUtilTest.java b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/ZipUtilTest.java
new file mode 100644
index 0000000..ed5c635
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/java/com/alipay/sofa/jraft/rhea/util/ZipUtilTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.zip.Checksum;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.util.CRC64;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class ZipUtilTest {
+
+    private File sourceDir;
+
+    @Before
+    public void startup() throws IOException {
+        this.sourceDir = new File("zip_test");
+        if (this.sourceDir.exists()) {
+            FileUtils.forceDelete(this.sourceDir);
+        }
+        FileUtils.forceMkdir(this.sourceDir);
+        final File f1 = Paths.get(this.sourceDir.getAbsolutePath(), "f1").toFile();
+        FileUtils.write(f1, "f1");
+        final File d1 = Paths.get(this.sourceDir.getAbsolutePath(), "d1").toFile();
+        FileUtils.forceMkdir(d1);
+        final File f11 = Paths.get(d1.getAbsolutePath(), "f11").toFile();
+        FileUtils.write(f11, "f11");
+
+        final File d2 = Paths.get(d1.getAbsolutePath(), "d2").toFile();
+        FileUtils.forceMkdir(d2);
+
+        final File d3 = Paths.get(d2.getAbsolutePath(), "d3").toFile();
+        FileUtils.forceMkdir(d3);
+
+        final File f31 = Paths.get(d3.getAbsolutePath(), "f31").toFile();
+        FileUtils.write(f31, "f32");
+    }
+
+    @After
+    public void teardown() throws IOException {
+        FileUtils.forceDelete(this.sourceDir);
+    }
+
+    @Test
+    public void zipTest() throws IOException {
+        final String rootPath = this.sourceDir.toPath().toAbsolutePath().getParent().toString();
+        final Path outPath = Paths.get(rootPath, "kv.zip");
+        final Checksum c1 = new CRC64();
+        ZipUtil.compress(rootPath, "zip_test", outPath.toString(), c1);
+
+        System.out.println(Long.toHexString(c1.getValue()));
+
+        final Checksum c2 = new CRC64();
+        ZipUtil.decompress(Paths.get(rootPath, "kv.zip").toString(), rootPath, c2);
+
+        Assert.assertEquals(c1.getValue(), c2.getValue());
+
+        FileUtils.forceDelete(outPath.toFile());
+    }
+}
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_1.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_1.yaml
new file mode 100644
index 0000000..56c4d5c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_1.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8181
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_2.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_2.yaml
new file mode 100644
index 0000000..e7f8a13
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_2.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8182
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_3.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_3.yaml
new file mode 100644
index 0000000..0572b86
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_cluster_3.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8183
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_1.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_1.yaml
new file mode 100644
index 0000000..dc5ccaa
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_1.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8181
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_2.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_2.yaml
new file mode 100644
index 0000000..2ca04ee
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_2.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8182
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_3.yaml b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_3.yaml
new file mode 100644
index 0000000..23199bf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/benchmark/conf/rhea_test_cluster_3.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8183
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_1.yaml b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_1.yaml
new file mode 100644
index 0000000..dc83692
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_1.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18181
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_2.yaml b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_2.yaml
new file mode 100644
index 0000000..17c2666
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_2.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18182
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_3.yaml b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_3.yaml
new file mode 100644
index 0000000..d688642
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/conf/rhea_test_cluster_3.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18183
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/log4j2.xml b/jraft-rheakv/rheakv-core/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..25b3a30
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_1.yaml b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_1.yaml
new file mode 100644
index 0000000..65df65d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_1.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18181
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_2.yaml b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_2.yaml
new file mode 100644
index 0000000..c124fda
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_2.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18182
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_3.yaml b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_3.yaml
new file mode 100644
index 0000000..beeb5d5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/src/test/resources/pd_conf/rhea_pd_test_3.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18183
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler b/jraft-rheakv/rheakv-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
new file mode 100644
index 0000000..a2950a0
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/classes/META-INF/services/com.alipay.sofa.jraft.util.JRaftSignalHandler
@@ -0,0 +1,2 @@
+com.alipay.sofa.jraft.rhea.RheaKVDescribeSignalHandler
+com.alipay.sofa.jraft.rhea.RheaKVMetricsSignalHandler
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_getApproximateKeysInRange_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_getApproximateKeysInRange_jmhTest.java
new file mode 100644
index 0000000..697c135
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_getApproximateKeysInRange_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVApproximateBenchmark_jmhType;
+public final class RawKVApproximateBenchmark_getApproximateKeysInRange_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult getApproximateKeysInRange_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G = _jmh_tryInit_f_rawkvapproximatebenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            getApproximateKeysInRange_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvapproximatebenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvapproximatebenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvapproximatebenchmark0_G.readyTrial) {
+                            l_rawkvapproximatebenchmark0_G.tearDown();
+                            l_rawkvapproximatebenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvapproximatebenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvapproximatebenchmark0_G_backoff = 1;
+                    while (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvapproximatebenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvapproximatebenchmark0_G_backoff);
+                        l_rawkvapproximatebenchmark0_G_backoff = Math.max(1024, l_rawkvapproximatebenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvapproximatebenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "getApproximateKeysInRange", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getApproximateKeysInRange_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getApproximateKeysInRange_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G = _jmh_tryInit_f_rawkvapproximatebenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            getApproximateKeysInRange_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvapproximatebenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvapproximatebenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvapproximatebenchmark0_G.readyTrial) {
+                            l_rawkvapproximatebenchmark0_G.tearDown();
+                            l_rawkvapproximatebenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvapproximatebenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvapproximatebenchmark0_G_backoff = 1;
+                    while (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvapproximatebenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvapproximatebenchmark0_G_backoff);
+                        l_rawkvapproximatebenchmark0_G_backoff = Math.max(1024, l_rawkvapproximatebenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvapproximatebenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "getApproximateKeysInRange", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getApproximateKeysInRange_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getApproximateKeysInRange_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G = _jmh_tryInit_f_rawkvapproximatebenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            getApproximateKeysInRange_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rawkvapproximatebenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvapproximatebenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvapproximatebenchmark0_G.readyTrial) {
+                            l_rawkvapproximatebenchmark0_G.tearDown();
+                            l_rawkvapproximatebenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvapproximatebenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvapproximatebenchmark0_G_backoff = 1;
+                    while (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvapproximatebenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvapproximatebenchmark0_G_backoff);
+                        l_rawkvapproximatebenchmark0_G_backoff = Math.max(1024, l_rawkvapproximatebenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvapproximatebenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "getApproximateKeysInRange", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getApproximateKeysInRange_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getApproximateKeysInRange_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G = _jmh_tryInit_f_rawkvapproximatebenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            getApproximateKeysInRange_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rawkvapproximatebenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvapproximatebenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvapproximatebenchmark0_G.readyTrial) {
+                            l_rawkvapproximatebenchmark0_G.tearDown();
+                            l_rawkvapproximatebenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvapproximatebenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvapproximatebenchmark0_G_backoff = 1;
+                    while (RawKVApproximateBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvapproximatebenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvapproximatebenchmark0_G_backoff);
+                        l_rawkvapproximatebenchmark0_G_backoff = Math.max(1024, l_rawkvapproximatebenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvapproximatebenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "getApproximateKeysInRange", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getApproximateKeysInRange_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RawKVApproximateBenchmark_jmhType l_rawkvapproximatebenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rawkvapproximatebenchmark0_G.getApproximateKeysInRange();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RawKVApproximateBenchmark_jmhType f_rawkvapproximatebenchmark0_G;
+    
+    RawKVApproximateBenchmark_jmhType _jmh_tryInit_f_rawkvapproximatebenchmark0_G(InfraControl control) throws Throwable {
+        RawKVApproximateBenchmark_jmhType val = f_rawkvapproximatebenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rawkvapproximatebenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RawKVApproximateBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rawkvapproximatebenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType.java
new file mode 100644
index 0000000..0f3e4de
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVApproximateBenchmark_jmhType extends RawKVApproximateBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..d09cdac
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVApproximateBenchmark;
+public class RawKVApproximateBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVApproximateBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..eb7a7be
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class RawKVApproximateBenchmark_jmhType_B2 extends RawKVApproximateBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVApproximateBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..d5e447c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVApproximateBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVApproximateBenchmark_jmhType_B3 extends RawKVApproximateBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_get_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_get_jmhTest.java
new file mode 100644
index 0000000..c8b458e
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_get_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVGetBenchmark_jmhType;
+public final class RawKVGetBenchmark_get_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult get_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G = _jmh_tryInit_f_rawkvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            get_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvgetbenchmark0_G.readyTrial) {
+                            l_rawkvgetbenchmark0_G.tearDown();
+                            l_rawkvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvgetbenchmark0_G_backoff = 1;
+                    while (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvgetbenchmark0_G_backoff);
+                        l_rawkvgetbenchmark0_G_backoff = Math.max(1024, l_rawkvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "get", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvgetbenchmark0_G.get();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G = _jmh_tryInit_f_rawkvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            get_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvgetbenchmark0_G.readyTrial) {
+                            l_rawkvgetbenchmark0_G.tearDown();
+                            l_rawkvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvgetbenchmark0_G_backoff = 1;
+                    while (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvgetbenchmark0_G_backoff);
+                        l_rawkvgetbenchmark0_G_backoff = Math.max(1024, l_rawkvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "get", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvgetbenchmark0_G.get();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G = _jmh_tryInit_f_rawkvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            get_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rawkvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvgetbenchmark0_G.readyTrial) {
+                            l_rawkvgetbenchmark0_G.tearDown();
+                            l_rawkvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvgetbenchmark0_G_backoff = 1;
+                    while (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvgetbenchmark0_G_backoff);
+                        l_rawkvgetbenchmark0_G_backoff = Math.max(1024, l_rawkvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "get", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rawkvgetbenchmark0_G.get();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G = _jmh_tryInit_f_rawkvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            get_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rawkvgetbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvgetbenchmark0_G.readyTrial) {
+                            l_rawkvgetbenchmark0_G.tearDown();
+                            l_rawkvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvgetbenchmark0_G_backoff = 1;
+                    while (RawKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvgetbenchmark0_G_backoff);
+                        l_rawkvgetbenchmark0_G_backoff = Math.max(1024, l_rawkvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvgetbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "get", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RawKVGetBenchmark_jmhType l_rawkvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rawkvgetbenchmark0_G.get();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RawKVGetBenchmark_jmhType f_rawkvgetbenchmark0_G;
+    
+    RawKVGetBenchmark_jmhType _jmh_tryInit_f_rawkvgetbenchmark0_G(InfraControl control) throws Throwable {
+        RawKVGetBenchmark_jmhType val = f_rawkvgetbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rawkvgetbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RawKVGetBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rawkvgetbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType.java
new file mode 100644
index 0000000..99efc0f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVGetBenchmark_jmhType extends RawKVGetBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..42aace2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVGetBenchmark;
+public class RawKVGetBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVGetBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..6280501
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class RawKVGetBenchmark_jmhType_B2 extends RawKVGetBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVGetBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..7c80efe
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVGetBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVGetBenchmark_jmhType_B3 extends RawKVGetBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType.java
new file mode 100644
index 0000000..465e2e4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVPutBenchmark_jmhType extends RawKVPutBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..fd7f09d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVPutBenchmark;
+public class RawKVPutBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVPutBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..31002af
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class RawKVPutBenchmark_jmhType_B2 extends RawKVPutBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RawKVPutBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..04a3c76
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+public class RawKVPutBenchmark_jmhType_B3 extends RawKVPutBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_put_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_put_jmhTest.java
new file mode 100644
index 0000000..5a66681
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/raw/generated/RawKVPutBenchmark_put_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.raw.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVPutBenchmark_jmhType;
+public final class RawKVPutBenchmark_put_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult put_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G = _jmh_tryInit_f_rawkvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            put_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvputbenchmark0_G.readyTrial) {
+                            l_rawkvputbenchmark0_G.tearDown();
+                            l_rawkvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvputbenchmark0_G_backoff = 1;
+                    while (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvputbenchmark0_G_backoff);
+                        l_rawkvputbenchmark0_G_backoff = Math.max(1024, l_rawkvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "put", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvputbenchmark0_G.put();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G = _jmh_tryInit_f_rawkvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            put_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rawkvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvputbenchmark0_G.readyTrial) {
+                            l_rawkvputbenchmark0_G.tearDown();
+                            l_rawkvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvputbenchmark0_G_backoff = 1;
+                    while (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvputbenchmark0_G_backoff);
+                        l_rawkvputbenchmark0_G_backoff = Math.max(1024, l_rawkvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "put", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rawkvputbenchmark0_G.put();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G = _jmh_tryInit_f_rawkvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rawkvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            put_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rawkvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rawkvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvputbenchmark0_G.readyTrial) {
+                            l_rawkvputbenchmark0_G.tearDown();
+                            l_rawkvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvputbenchmark0_G_backoff = 1;
+                    while (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvputbenchmark0_G_backoff);
+                        l_rawkvputbenchmark0_G_backoff = Math.max(1024, l_rawkvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "put", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rawkvputbenchmark0_G.put();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G = _jmh_tryInit_f_rawkvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            put_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rawkvputbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rawkvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rawkvputbenchmark0_G.readyTrial) {
+                            l_rawkvputbenchmark0_G.tearDown();
+                            l_rawkvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rawkvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rawkvputbenchmark0_G_backoff = 1;
+                    while (RawKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rawkvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rawkvputbenchmark0_G_backoff);
+                        l_rawkvputbenchmark0_G_backoff = Math.max(1024, l_rawkvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rawkvputbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "put", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RawKVPutBenchmark_jmhType l_rawkvputbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rawkvputbenchmark0_G.put();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RawKVPutBenchmark_jmhType f_rawkvputbenchmark0_G;
+    
+    RawKVPutBenchmark_jmhType _jmh_tryInit_f_rawkvputbenchmark0_G(InfraControl control) throws Throwable {
+        RawKVPutBenchmark_jmhType val = f_rawkvputbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rawkvputbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RawKVPutBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rawkvputbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_getReadOnlySafe_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_getReadOnlySafe_jmhTest.java
new file mode 100644
index 0000000..843a182
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_getReadOnlySafe_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVGetBenchmark_jmhType;
+public final class RheaKVGetBenchmark_getReadOnlySafe_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult getReadOnlySafe_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            getReadOnlySafe_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "getReadOnlySafe", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getReadOnlySafe_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvgetbenchmark0_G.getReadOnlySafe();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getReadOnlySafe_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            getReadOnlySafe_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "getReadOnlySafe", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getReadOnlySafe_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvgetbenchmark0_G.getReadOnlySafe();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getReadOnlySafe_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            getReadOnlySafe_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.getReadOnlySafe();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "getReadOnlySafe", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getReadOnlySafe_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rheakvgetbenchmark0_G.getReadOnlySafe();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult getReadOnlySafe_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            getReadOnlySafe_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rheakvgetbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "getReadOnlySafe", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void getReadOnlySafe_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rheakvgetbenchmark0_G.getReadOnlySafe();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RheaKVGetBenchmark_jmhType f_rheakvgetbenchmark0_G;
+    
+    RheaKVGetBenchmark_jmhType _jmh_tryInit_f_rheakvgetbenchmark0_G(InfraControl control) throws Throwable {
+        RheaKVGetBenchmark_jmhType val = f_rheakvgetbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rheakvgetbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RheaKVGetBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rheakvgetbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_get_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_get_jmhTest.java
new file mode 100644
index 0000000..a319b24
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_get_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVGetBenchmark_jmhType;
+public final class RheaKVGetBenchmark_get_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult get_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            get_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "get", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvgetbenchmark0_G.get();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            get_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "get", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvgetbenchmark0_G.get();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvgetbenchmark0_G.get();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            get_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rheakvgetbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvgetbenchmark0_G.get();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "get", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rheakvgetbenchmark0_G.get();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult get_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G = _jmh_tryInit_f_rheakvgetbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            get_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rheakvgetbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvgetbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvgetbenchmark0_G.readyTrial) {
+                            l_rheakvgetbenchmark0_G.tearDown();
+                            l_rheakvgetbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvgetbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvgetbenchmark0_G_backoff = 1;
+                    while (RheaKVGetBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvgetbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvgetbenchmark0_G_backoff);
+                        l_rheakvgetbenchmark0_G_backoff = Math.max(1024, l_rheakvgetbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvgetbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "get", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void get_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RheaKVGetBenchmark_jmhType l_rheakvgetbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rheakvgetbenchmark0_G.get();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RheaKVGetBenchmark_jmhType f_rheakvgetbenchmark0_G;
+    
+    RheaKVGetBenchmark_jmhType _jmh_tryInit_f_rheakvgetbenchmark0_G(InfraControl control) throws Throwable {
+        RheaKVGetBenchmark_jmhType val = f_rheakvgetbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rheakvgetbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RheaKVGetBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rheakvgetbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType.java
new file mode 100644
index 0000000..80a80c4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+public class RheaKVGetBenchmark_jmhType extends RheaKVGetBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..5506d6d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+import com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVGetBenchmark;
+public class RheaKVGetBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVGetBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..9d6cab8
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class RheaKVGetBenchmark_jmhType_B2 extends RheaKVGetBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVGetBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..05c271f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVGetBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+public class RheaKVGetBenchmark_jmhType_B3 extends RheaKVGetBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType.java
new file mode 100644
index 0000000..e0e7e60
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+public class RheaKVPutBenchmark_jmhType extends RheaKVPutBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..fd5cb6f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+import com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVPutBenchmark;
+public class RheaKVPutBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVPutBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..8a1199f
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class RheaKVPutBenchmark_jmhType_B2 extends RheaKVPutBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(RheaKVPutBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..0312602
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+public class RheaKVPutBenchmark_jmhType_B3 extends RheaKVPutBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_put_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_put_jmhTest.java
new file mode 100644
index 0000000..5b34a36
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/benchmark/rhea/generated/RheaKVPutBenchmark_put_jmhTest.java
@@ -0,0 +1,451 @@
+package com.alipay.sofa.jraft.rhea.benchmark.rhea.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVPutBenchmark_jmhType;
+public final class RheaKVPutBenchmark_put_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult put_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G = _jmh_tryInit_f_rheakvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            put_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvputbenchmark0_G.readyTrial) {
+                            l_rheakvputbenchmark0_G.tearDown();
+                            l_rheakvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvputbenchmark0_G_backoff = 1;
+                    while (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvputbenchmark0_G_backoff);
+                        l_rheakvputbenchmark0_G_backoff = Math.max(1024, l_rheakvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "put", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvputbenchmark0_G.put();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G = _jmh_tryInit_f_rheakvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            put_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_rheakvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvputbenchmark0_G.readyTrial) {
+                            l_rheakvputbenchmark0_G.tearDown();
+                            l_rheakvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvputbenchmark0_G_backoff = 1;
+                    while (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvputbenchmark0_G_backoff);
+                        l_rheakvputbenchmark0_G_backoff = Math.max(1024, l_rheakvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "put", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_rheakvputbenchmark0_G.put();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G = _jmh_tryInit_f_rheakvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_rheakvputbenchmark0_G.put();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            put_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_rheakvputbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_rheakvputbenchmark0_G.put();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                if (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvputbenchmark0_G.readyTrial) {
+                            l_rheakvputbenchmark0_G.tearDown();
+                            l_rheakvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvputbenchmark0_G_backoff = 1;
+                    while (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvputbenchmark0_G_backoff);
+                        l_rheakvputbenchmark0_G_backoff = Math.max(1024, l_rheakvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvputbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "put", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_rheakvputbenchmark0_G.put();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult put_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G = _jmh_tryInit_f_rheakvputbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            put_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_rheakvputbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                if (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.compareAndSet(l_rheakvputbenchmark0_G, 0, 1)) {
+                    try {
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (l_rheakvputbenchmark0_G.readyTrial) {
+                            l_rheakvputbenchmark0_G.tearDown();
+                            l_rheakvputbenchmark0_G.readyTrial = false;
+                        }
+                    } catch (Throwable t) {
+                        control.isFailing = true;
+                        throw t;
+                    } finally {
+                        RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.set(l_rheakvputbenchmark0_G, 0);
+                    }
+                } else {
+                    long l_rheakvputbenchmark0_G_backoff = 1;
+                    while (RheaKVPutBenchmark_jmhType.tearTrialMutexUpdater.get(l_rheakvputbenchmark0_G) == 1) {
+                        TimeUnit.MILLISECONDS.sleep(l_rheakvputbenchmark0_G_backoff);
+                        l_rheakvputbenchmark0_G_backoff = Math.max(1024, l_rheakvputbenchmark0_G_backoff * 2);
+                        if (control.isFailing) throw new FailureAssistException();
+                        if (Thread.interrupted()) throw new InterruptedException();
+                    }
+                }
+                synchronized(this.getClass()) {
+                    f_rheakvputbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "put", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void put_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, RheaKVPutBenchmark_jmhType l_rheakvputbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_rheakvputbenchmark0_G.put();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile RheaKVPutBenchmark_jmhType f_rheakvputbenchmark0_G;
+    
+    RheaKVPutBenchmark_jmhType _jmh_tryInit_f_rheakvputbenchmark0_G(InfraControl control) throws Throwable {
+        RheaKVPutBenchmark_jmhType val = f_rheakvputbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_rheakvputbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new RheaKVPutBenchmark_jmhType();
+            val.setup();
+            val.readyTrial = true;
+            f_rheakvputbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_int64_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_int64_jmhTest.java
new file mode 100644
index 0000000..248fd43
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_int64_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.util.generated.VarIntsBenchmark_jmhType;
+public final class VarIntsBenchmark_int64_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult int64_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.int64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int64_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.int64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "int64", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void int64_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_varintsbenchmark0_G.int64();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult int64_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.int64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int64_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.int64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "int64", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void int64_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_varintsbenchmark0_G.int64();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult int64_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.int64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            int64_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.int64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "int64", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void int64_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_varintsbenchmark0_G.int64();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult int64_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            int64_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_varintsbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "int64", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void int64_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_varintsbenchmark0_G.int64();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile VarIntsBenchmark_jmhType f_varintsbenchmark0_G;
+    
+    VarIntsBenchmark_jmhType _jmh_tryInit_f_varintsbenchmark0_G(InfraControl control) throws Throwable {
+        VarIntsBenchmark_jmhType val = f_varintsbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_varintsbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new VarIntsBenchmark_jmhType();
+            val.readyTrial = true;
+            f_varintsbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType.java
new file mode 100644
index 0000000..3fef0b4
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType.java
@@ -0,0 +1,4 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+public class VarIntsBenchmark_jmhType extends VarIntsBenchmark_jmhType_B3 {
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B1.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B1.java
new file mode 100644
index 0000000..7a43173
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B1.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+import com.alipay.sofa.jraft.rhea.util.VarIntsBenchmark;
+public class VarIntsBenchmark_jmhType_B1 extends com.alipay.sofa.jraft.rhea.util.VarIntsBenchmark {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B2.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B2.java
new file mode 100644
index 0000000..873b4c2
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B2.java
@@ -0,0 +1,22 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+public class VarIntsBenchmark_jmhType_B2 extends VarIntsBenchmark_jmhType_B1 {
+    public volatile int setupTrialMutex;
+    public volatile int tearTrialMutex;
+    public final static AtomicIntegerFieldUpdater setupTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "setupTrialMutex");
+    public final static AtomicIntegerFieldUpdater tearTrialMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "tearTrialMutex");
+
+    public volatile int setupIterationMutex;
+    public volatile int tearIterationMutex;
+    public final static AtomicIntegerFieldUpdater setupIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "setupIterationMutex");
+    public final static AtomicIntegerFieldUpdater tearIterationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "tearIterationMutex");
+
+    public volatile int setupInvocationMutex;
+    public volatile int tearInvocationMutex;
+    public final static AtomicIntegerFieldUpdater setupInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "setupInvocationMutex");
+    public final static AtomicIntegerFieldUpdater tearInvocationMutexUpdater = AtomicIntegerFieldUpdater.newUpdater(VarIntsBenchmark_jmhType_B2.class, "tearInvocationMutex");
+
+    public volatile boolean readyTrial;
+    public volatile boolean readyIteration;
+    public volatile boolean readyInvocation;
+}
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B3.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B3.java
new file mode 100644
index 0000000..60db902
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_jmhType_B3.java
@@ -0,0 +1,20 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+public class VarIntsBenchmark_jmhType_B3 extends VarIntsBenchmark_jmhType_B2 {
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_varInt64_jmhTest.java b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_varInt64_jmhTest.java
new file mode 100644
index 0000000..636768d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/generated-test-sources/test-annotations/com/alipay/sofa/jraft/rhea/util/generated/VarIntsBenchmark_varInt64_jmhTest.java
@@ -0,0 +1,362 @@
+package com.alipay.sofa.jraft.rhea.util.generated;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.Collection;
+import java.util.ArrayList;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.CompilerControl;
+import org.openjdk.jmh.runner.InfraControl;
+import org.openjdk.jmh.infra.ThreadParams;
+import org.openjdk.jmh.results.BenchmarkTaskResult;
+import org.openjdk.jmh.results.Result;
+import org.openjdk.jmh.results.ThroughputResult;
+import org.openjdk.jmh.results.AverageTimeResult;
+import org.openjdk.jmh.results.SampleTimeResult;
+import org.openjdk.jmh.results.SingleShotResult;
+import org.openjdk.jmh.util.SampleBuffer;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.results.RawResults;
+import org.openjdk.jmh.results.ResultRole;
+import java.lang.reflect.Field;
+import org.openjdk.jmh.infra.BenchmarkParams;
+import org.openjdk.jmh.infra.IterationParams;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.infra.Control;
+import org.openjdk.jmh.results.ScalarResult;
+import org.openjdk.jmh.results.AggregationPolicy;
+import org.openjdk.jmh.runner.FailureAssistException;
+
+import com.alipay.sofa.jraft.rhea.util.generated.VarIntsBenchmark_jmhType;
+public final class VarIntsBenchmark_varInt64_jmhTest {
+
+    boolean p000, p001, p002, p003, p004, p005, p006, p007, p008, p009, p010, p011, p012, p013, p014, p015;
+    boolean p016, p017, p018, p019, p020, p021, p022, p023, p024, p025, p026, p027, p028, p029, p030, p031;
+    boolean p032, p033, p034, p035, p036, p037, p038, p039, p040, p041, p042, p043, p044, p045, p046, p047;
+    boolean p048, p049, p050, p051, p052, p053, p054, p055, p056, p057, p058, p059, p060, p061, p062, p063;
+    boolean p064, p065, p066, p067, p068, p069, p070, p071, p072, p073, p074, p075, p076, p077, p078, p079;
+    boolean p080, p081, p082, p083, p084, p085, p086, p087, p088, p089, p090, p091, p092, p093, p094, p095;
+    boolean p096, p097, p098, p099, p100, p101, p102, p103, p104, p105, p106, p107, p108, p109, p110, p111;
+    boolean p112, p113, p114, p115, p116, p117, p118, p119, p120, p121, p122, p123, p124, p125, p126, p127;
+    boolean p128, p129, p130, p131, p132, p133, p134, p135, p136, p137, p138, p139, p140, p141, p142, p143;
+    boolean p144, p145, p146, p147, p148, p149, p150, p151, p152, p153, p154, p155, p156, p157, p158, p159;
+    boolean p160, p161, p162, p163, p164, p165, p166, p167, p168, p169, p170, p171, p172, p173, p174, p175;
+    boolean p176, p177, p178, p179, p180, p181, p182, p183, p184, p185, p186, p187, p188, p189, p190, p191;
+    boolean p192, p193, p194, p195, p196, p197, p198, p199, p200, p201, p202, p203, p204, p205, p206, p207;
+    boolean p208, p209, p210, p211, p212, p213, p214, p215, p216, p217, p218, p219, p220, p221, p222, p223;
+    boolean p224, p225, p226, p227, p228, p229, p230, p231, p232, p233, p234, p235, p236, p237, p238, p239;
+    boolean p240, p241, p242, p243, p244, p245, p246, p247, p248, p249, p250, p251, p252, p253, p254, p255;
+    int startRndMask;
+    BenchmarkParams benchmarkParams;
+    IterationParams iterationParams;
+    ThreadParams threadParams;
+    Blackhole blackhole;
+    Control notifyControl;
+
+    public BenchmarkTaskResult varInt64_Throughput(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.varInt64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            varInt64_thrpt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.varInt64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new ThroughputResult(ResultRole.PRIMARY, "varInt64", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void varInt64_thrpt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_varintsbenchmark0_G.varInt64();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult varInt64_AverageTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.varInt64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            varInt64_avgt_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.varInt64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps;
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            res.measuredOps /= batchSize;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new AverageTimeResult(ResultRole.PRIMARY, "varInt64", res.measuredOps, res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void varInt64_avgt_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long operations = 0;
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        do {
+            l_varintsbenchmark0_G.varInt64();
+            operations++;
+        } while(!control.isDone);
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult varInt64_SampleTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            RawResults res = new RawResults();
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            control.announceWarmupReady();
+            while (control.warmupShouldWait) {
+                l_varintsbenchmark0_G.varInt64();
+                res.allOps++;
+            }
+
+            notifyControl.startMeasurement = true;
+            int targetSamples = (int) (control.getDuration(TimeUnit.MILLISECONDS) * 20); // at max, 20 timestamps per millisecond
+            int batchSize = iterationParams.getBatchSize();
+            int opsPerInv = benchmarkParams.getOpsPerInvocation();
+            SampleBuffer buffer = new SampleBuffer();
+            varInt64_sample_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, buffer, targetSamples, opsPerInv, batchSize, l_varintsbenchmark0_G);
+            notifyControl.stopMeasurement = true;
+            control.announceWarmdownReady();
+            try {
+                while (control.warmdownShouldWait) {
+                    l_varintsbenchmark0_G.varInt64();
+                    res.allOps++;
+                }
+                control.preTearDown();
+            } catch (InterruptedException ie) {
+                control.preTearDownForce();
+            }
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            res.allOps += res.measuredOps * batchSize;
+            res.allOps *= opsPerInv;
+            res.allOps /= batchSize;
+            res.measuredOps *= opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(res.allOps, res.measuredOps);
+            results.add(new SampleTimeResult(ResultRole.PRIMARY, "varInt64", buffer, benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void varInt64_sample_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, SampleBuffer buffer, int targetSamples, long opsPerInv, int batchSize, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        long operations = 0;
+        int rnd = (int)System.nanoTime();
+        int rndMask = startRndMask;
+        long time = 0;
+        int currentStride = 0;
+        do {
+            rnd = (rnd * 1664525 + 1013904223);
+            boolean sample = (rnd & rndMask) == 0;
+            if (sample) {
+                time = System.nanoTime();
+            }
+            for (int b = 0; b < batchSize; b++) {
+                if (control.volatileSpoiler) return;
+                l_varintsbenchmark0_G.varInt64();
+            }
+            if (sample) {
+                buffer.add((System.nanoTime() - time) / opsPerInv);
+                if (currentStride++ > targetSamples) {
+                    buffer.half();
+                    currentStride = 0;
+                    rndMask = (rndMask << 1) + 1;
+                }
+            }
+            operations++;
+        } while(!control.isDone);
+        startRndMask = Math.max(startRndMask, rndMask);
+        result.realTime = realTime;
+        result.measuredOps = operations;
+    }
+
+
+    public BenchmarkTaskResult varInt64_SingleShotTime(InfraControl control, ThreadParams threadParams) throws Throwable {
+        this.benchmarkParams = control.benchmarkParams;
+        this.iterationParams = control.iterationParams;
+        this.threadParams    = threadParams;
+        this.notifyControl   = control.notifyControl;
+        if (this.blackhole == null) {
+            this.blackhole = new Blackhole("Today's password is swordfish. I understand instantiating Blackholes directly is dangerous.");
+        }
+        if (threadParams.getSubgroupIndex() == 0) {
+            VarIntsBenchmark_jmhType l_varintsbenchmark0_G = _jmh_tryInit_f_varintsbenchmark0_G(control);
+
+            control.preSetup();
+
+
+            notifyControl.startMeasurement = true;
+            RawResults res = new RawResults();
+            int batchSize = iterationParams.getBatchSize();
+            varInt64_ss_jmhStub(control, res, benchmarkParams, iterationParams, threadParams, blackhole, notifyControl, startRndMask, batchSize, l_varintsbenchmark0_G);
+            control.preTearDown();
+
+            if (control.isLastIteration()) {
+                synchronized(this.getClass()) {
+                    f_varintsbenchmark0_G = null;
+                }
+            }
+            int opsPerInv = control.benchmarkParams.getOpsPerInvocation();
+            long totalOps = opsPerInv;
+            BenchmarkTaskResult results = new BenchmarkTaskResult(totalOps, totalOps);
+            results.add(new SingleShotResult(ResultRole.PRIMARY, "varInt64", res.getTime(), benchmarkParams.getTimeUnit()));
+            this.blackhole.evaporate("Yes, I am Stephen Hawking, and know a thing or two about black holes.");
+            return results;
+        } else
+            throw new IllegalStateException("Harness failed to distribute threads among groups properly");
+    }
+
+    public static void varInt64_ss_jmhStub(InfraControl control, RawResults result, BenchmarkParams benchmarkParams, IterationParams iterationParams, ThreadParams threadParams, Blackhole blackhole, Control notifyControl, int startRndMask, int batchSize, VarIntsBenchmark_jmhType l_varintsbenchmark0_G) throws Throwable {
+        long realTime = 0;
+        result.startTime = System.nanoTime();
+        for (int b = 0; b < batchSize; b++) {
+            if (control.volatileSpoiler) return;
+            l_varintsbenchmark0_G.varInt64();
+        }
+        result.stopTime = System.nanoTime();
+        result.realTime = realTime;
+    }
+
+    
+    static volatile VarIntsBenchmark_jmhType f_varintsbenchmark0_G;
+    
+    VarIntsBenchmark_jmhType _jmh_tryInit_f_varintsbenchmark0_G(InfraControl control) throws Throwable {
+        VarIntsBenchmark_jmhType val = f_varintsbenchmark0_G;
+        if (val != null) {
+            return val;
+        }
+        synchronized(this.getClass()) {
+            try {
+            if (control.isFailing) throw new FailureAssistException();
+            val = f_varintsbenchmark0_G;
+            if (val != null) {
+                return val;
+            }
+            val = new VarIntsBenchmark_jmhType();
+            val.readyTrial = true;
+            f_varintsbenchmark0_G = val;
+            } catch (Throwable t) {
+                control.isFailing = true;
+                throw t;
+            }
+        }
+        return val;
+    }
+
+
+}
+
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/META-INF/BenchmarkList b/jraft-rheakv/rheakv-core/target/test-classes/META-INF/BenchmarkList
new file mode 100644
index 0000000..67a3eab
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/META-INF/BenchmarkList
@@ -0,0 +1,8 @@
+JMH S 58 com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVGetBenchmark S 80 com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVGetBenchmark_get_jmhTest S 3 get S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 66 com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVApproximateBenchmark S 110 com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVApproximateBenchmark_getApproximateKeysInRange_jmhTest S 25 getApproximateKeysInRange S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rhea.util.VarIntsBenchmark S 75 com.alipay.sofa.jraft.rhea.util.generated.VarIntsBenchmark_varInt64_jmhTest S 8 varInt64 S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 58 com.alipay.sofa.jraft.rhea.benchmark.raw.RawKVPutBenchmark S 80 com.alipay.sofa.jraft.rhea.benchmark.raw.generated.RawKVPutBenchmark_put_jmhTest S 3 put S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 48 com.alipay.sofa.jraft.rhea.util.VarIntsBenchmark S 72 com.alipay.sofa.jraft.rhea.util.generated.VarIntsBenchmark_int64_jmhTest S 5 int64 S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 11 NANOSECONDS E E 
+JMH S 60 com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVGetBenchmark S 94 com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVGetBenchmark_getReadOnlySafe_jmhTest S 15 getReadOnlySafe S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 60 com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVPutBenchmark S 82 com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVPutBenchmark_put_jmhTest S 3 put S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
+JMH S 60 com.alipay.sofa.jraft.rhea.benchmark.rhea.RheaKVGetBenchmark S 82 com.alipay.sofa.jraft.rhea.benchmark.rhea.generated.RheaKVGetBenchmark_get_jmhTest S 3 get S 3 All E A 1 1 1 E E E E E E E E E E E E E E U 12 MILLISECONDS E E 
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/META-INF/CompilerHints b/jraft-rheakv/rheakv-core/target/test-classes/META-INF/CompilerHints
new file mode 100644
index 0000000..54336b9
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/META-INF/CompilerHints
@@ -0,0 +1,23 @@
+dontinline,*.*_all_jmhStub
+dontinline,*.*_avgt_jmhStub
+dontinline,*.*_sample_jmhStub
+dontinline,*.*_ss_jmhStub
+dontinline,*.*_thrpt_jmhStub
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.getApproximateKeysInRange
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.setup
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVApproximateBenchmark.tearDown
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.get
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.setup
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVGetBenchmark.tearDown
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.put
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.setup
+inline,com/alipay/sofa/jraft/rhea/benchmark/raw/RawKVPutBenchmark.tearDown
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.get
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.getReadOnlySafe
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.setup
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVGetBenchmark.tearDown
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.put
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.setup
+inline,com/alipay/sofa/jraft/rhea/benchmark/rhea/RheaKVPutBenchmark.tearDown
+inline,com/alipay/sofa/jraft/rhea/util/VarIntsBenchmark.int64
+inline,com/alipay/sofa/jraft/rhea/util/VarIntsBenchmark.varInt64
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_1.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_1.yaml
new file mode 100644
index 0000000..56c4d5c
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_1.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8181
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_2.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_2.yaml
new file mode 100644
index 0000000..e7f8a13
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_2.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8182
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_3.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_3.yaml
new file mode 100644
index 0000000..0572b86
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_cluster_3.yaml
@@ -0,0 +1,19 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_benchmark
+
+placementDriverOptions:
+  fake: true
+
+storeEngineOptions:
+  rocksDBOptions:
+    sync: false
+    dbPath: benchmark_rhea_db/
+  raftDataPath: benchmark_rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8183
+  regionEngineOptionsList:
+    - { regionId: 1, nodeOptions: { raftOptions: { sync: false} } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_1.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_1.yaml
new file mode 100644
index 0000000..dc5ccaa
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_1.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8181
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_2.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_2.yaml
new file mode 100644
index 0000000..2ca04ee
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_2.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8182
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_3.yaml b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_3.yaml
new file mode 100644
index 0000000..23199bf
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/benchmark/conf/rhea_test_cluster_3.yaml
@@ -0,0 +1,22 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 8183
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g,  endKey: t, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 3, startKey: t            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+
+initialServerList: 127.0.0.1:8181,127.0.0.1:8182,127.0.0.1:8183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_1.yaml b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_1.yaml
new file mode 100644
index 0000000..dc83692
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_1.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18181
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_2.yaml b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_2.yaml
new file mode 100644
index 0000000..17c2666
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_2.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18182
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_3.yaml b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_3.yaml
new file mode 100644
index 0000000..d688642
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/conf/rhea_test_cluster_3.yaml
@@ -0,0 +1,24 @@
+##RheaKVStoreOptions
+---
+clusterName: rhea_test
+
+placementDriverOptions:
+  fake: true
+  cliOptions:
+    rpcProcessorThreadPoolSize: 4
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_db/
+  raftDataPath: rhea_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18183
+  regionEngineOptionsList:
+    - { regionId: 1,               endKey: g, nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+    - { regionId: 2, startKey: g            , nodeOptions: { timerPoolSize: 1, rpcProcessorThreadPoolSize: 4 } }
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner
+failoverRetries: 10
+futureTimeoutMillis: 30000
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/log4j2.xml b/jraft-rheakv/rheakv-core/target/test-classes/log4j2.xml
new file mode 100644
index 0000000..25b3a30
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_1.yaml b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_1.yaml
new file mode 100644
index 0000000..65df65d
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_1.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18181
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_2.yaml b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_2.yaml
new file mode 100644
index 0000000..c124fda
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_2.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18182
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_3.yaml b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_3.yaml
new file mode 100644
index 0000000..beeb5d5
--- /dev/null
+++ b/jraft-rheakv/rheakv-core/target/test-classes/pd_conf/rhea_pd_test_3.yaml
@@ -0,0 +1,23 @@
+##RheaKVStoreOptions
+---
+clusterId: 111
+clusterName: rhea_pd_test
+
+placementDriverOptions:
+  fake: false
+  pdGroupId: pd_test--1
+  initialPdServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
+
+storeEngineOptions:
+  rocksDBOptions:
+    dbPath: rhea_pd_db/
+  raftDataPath: rhea_pd_raft/
+  serverAddress:
+    ip: 127.0.0.1
+    port: 18183
+  regionEngineOptionsList:
+    - { regionId: 1,   endKey: g, initialServerList: "127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183/learner"}
+    - { regionId: 2, startKey: g}
+  leastKeysOnSplit: 10
+
+initialServerList: 127.0.0.1:18181,127.0.0.1:18182,127.0.0.1:18183
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-pd/pom.xml b/jraft-rheakv/rheakv-pd/pom.xml
new file mode 100644
index 0000000..a6d4053
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/pom.xml
@@ -0,0 +1,47 @@
+
+
+    4.0.0
+    
+        jraft-rheakv
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+
+    jraft-rheakv-pd
+    rheakv-pd ${project.version}
+
+    
+        2.10.5.1
+        2.10.4
+    
+
+    
+        
+            ${project.groupId}
+            jraft-rheakv-core
+        
+
+        
+            junit
+            junit
+        
+        
+            org.hamcrest
+            hamcrest-library
+        
+        
+            org.mockito
+            mockito-all
+        
+        
+            com.fasterxml.jackson.dataformat
+            jackson-dataformat-yaml
+            ${jackson.dataformat.version}
+        
+        
+            com.fasterxml.jackson.core
+            jackson-databind
+            ${jackson.databind.version}
+        
+    
+
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/ClusterStatsManager.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/ClusterStatsManager.java
new file mode 100644
index 0000000..9a8a03a
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/ClusterStatsManager.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import com.alipay.remoting.util.ConcurrentHashSet;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class ClusterStatsManager {
+
+    private static final ConcurrentMap clusterStatsManagerTable = Maps
+                                                                                               .newConcurrentMapLong();
+
+    private final long                                            clusterId;
+    // Map>
+    private final ConcurrentMap>                  leaderTable              = Maps
+                                                                                               .newConcurrentMapLong();
+    // Map>
+    private final ConcurrentMap>  regionStatsTable         = Maps
+                                                                                               .newConcurrentMapLong();
+
+    private ClusterStatsManager(long clusterId) {
+        this.clusterId = clusterId;
+    }
+
+    public static ClusterStatsManager getInstance(final long clusterId) {
+        ClusterStatsManager instance = clusterStatsManagerTable.get(clusterId);
+        if (instance == null) {
+            final ClusterStatsManager newInstance = new ClusterStatsManager(clusterId);
+            instance = clusterStatsManagerTable.putIfAbsent(clusterId, newInstance);
+            if (instance == null) {
+                instance = newInstance;
+            }
+        }
+        return instance;
+    }
+
+    public long getClusterId() {
+        return clusterId;
+    }
+
+    public int regionSize() {
+        return this.regionStatsTable.size();
+    }
+
+    public void addOrUpdateLeader(final long storeId, final long regionId) {
+        Set regionTable = this.leaderTable.get(storeId);
+        if (regionTable == null) {
+            final Set newRegionTable = new ConcurrentHashSet<>();
+            regionTable = this.leaderTable.putIfAbsent(storeId, newRegionTable);
+            if (regionTable == null) {
+                regionTable = newRegionTable;
+            }
+        }
+        if (regionTable.add(regionId)) {
+            for (final Map.Entry> entry : this.leaderTable.entrySet()) {
+                if (storeId == entry.getKey()) {
+                    continue;
+                }
+                entry.getValue().remove(regionId);
+            }
+        }
+    }
+
+    // Looking for a model worker
+    public Pair, Integer /* leaderCount */> findModelWorkerStores(final int above) {
+        final Set>> values = this.leaderTable.entrySet();
+        if (values.isEmpty()) {
+            return Pair.of(Collections.emptySet(), 0);
+        }
+        final Map.Entry> modelWorker = Collections.max(values, (o1, o2) -> {
+            final int o1Val = o1.getValue() == null ? 0 : o1.getValue().size();
+            final int o2Val = o2.getValue() == null ? 0 : o2.getValue().size();
+            return Integer.compare(o1Val, o2Val);
+        });
+        final int maxLeaderCount = modelWorker.getValue().size();
+        if (maxLeaderCount <= above) {
+            return Pair.of(Collections.emptySet(), maxLeaderCount);
+        }
+        final Set modelWorkerStoreIds = new HashSet<>();
+        for (final Map.Entry> entry : values) {
+            if (entry.getValue().size() >= maxLeaderCount) {
+                modelWorkerStoreIds.add(entry.getKey());
+            }
+        }
+        return Pair.of(modelWorkerStoreIds, maxLeaderCount);
+    }
+
+    // Investigate who is lazy
+    public List> findLazyWorkerStores(final Collection storeCandidates) {
+        if (storeCandidates == null || storeCandidates.isEmpty()) {
+            return Collections.emptyList();
+        }
+        final Set>> values = this.leaderTable.entrySet();
+        if (values.isEmpty()) {
+            return Collections.emptyList();
+        }
+        final Map.Entry> lazyWorker = Collections.min(values, (o1, o2) -> {
+            final int o1Val = o1.getValue() == null ? 0 : o1.getValue().size();
+            final int o2Val = o2.getValue() == null ? 0 : o2.getValue().size();
+            return Integer.compare(o1Val, o2Val);
+        });
+        final int minLeaderCount = lazyWorker.getValue().size();
+        final List> lazyCandidates = Lists.newArrayList();
+        for (final Long storeId : storeCandidates) {
+            final Set regionTable = this.leaderTable.get(storeId);
+            int leaderCount = regionTable == null ? 0 : regionTable.size();
+            if (leaderCount <= minLeaderCount) {
+                lazyCandidates.add(Pair.of(storeId, leaderCount));
+            }
+        }
+        return lazyCandidates;
+    }
+
+    public void addOrUpdateRegionStats(final List> regionStatsList) {
+        for (final Pair p : regionStatsList) {
+            this.regionStatsTable.put(p.getKey().getId(), p);
+        }
+    }
+
+    public Pair findModelWorkerRegion() {
+        if (this.regionStatsTable.isEmpty()) {
+            return null;
+        }
+        return Collections.max(this.regionStatsTable.values(), (o1, o2) -> {
+            final long o1Val = o1.getValue().getApproximateKeys();
+            final long o2Val = o2.getValue().getApproximateKeys();
+            return Long.compare(o1Val, o2Val);
+        });
+    }
+
+    public static void invalidCache() {
+        for (final ClusterStatsManager manager : clusterStatsManagerTable.values()) {
+            manager.leaderTable.clear();
+            manager.regionStatsTable.clear();
+        }
+        clusterStatsManagerTable.clear();
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultMetadataStore.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultMetadataStore.java
new file mode 100644
index 0000000..f6465ba
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultMetadataStore.java
@@ -0,0 +1,302 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentMap;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.serialization.Serializer;
+import com.alipay.sofa.jraft.rhea.serialization.Serializers;
+import com.alipay.sofa.jraft.rhea.storage.KVEntry;
+import com.alipay.sofa.jraft.rhea.storage.LongSequence;
+import com.alipay.sofa.jraft.rhea.storage.Sequence;
+import com.alipay.sofa.jraft.rhea.util.ByteArray;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Maps;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.Strings;
+import com.alipay.sofa.jraft.util.Bits;
+import com.alipay.sofa.jraft.util.BytesUtil;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultMetadataStore implements MetadataStore {
+
+    private static final Logger                       LOG                  = LoggerFactory
+                                                                               .getLogger(DefaultMetadataStore.class);
+
+    private final ConcurrentMap storeSequenceMap     = Maps.newConcurrentMap();
+    private final ConcurrentMap regionSequenceMap    = Maps.newConcurrentMap();
+    private final ConcurrentMap>      clusterStoreIdsCache = Maps.newConcurrentMapLong();
+    private final Serializer                          serializer           = Serializers.getDefault();
+    private final RheaKVStore                         rheaKVStore;
+
+    public DefaultMetadataStore(RheaKVStore rheaKVStore) {
+        this.rheaKVStore = rheaKVStore;
+    }
+
+    @Override
+    public Cluster getClusterInfo(final long clusterId) {
+        final Set storeIds = getClusterIndex(clusterId);
+        if (storeIds == null) {
+            return null;
+        }
+        final List storeKeys = Lists.newArrayList();
+        for (final Long storeId : storeIds) {
+            final String storeInfoKey = MetadataKeyHelper.getStoreInfoKey(clusterId, storeId);
+            storeKeys.add(BytesUtil.writeUtf8(storeInfoKey));
+        }
+        final Map storeInfoBytes = this.rheaKVStore.bMultiGet(storeKeys);
+        final List stores = Lists.newArrayListWithCapacity(storeInfoBytes.size());
+        for (final byte[] storeBytes : storeInfoBytes.values()) {
+            final Store store = this.serializer.readObject(storeBytes, Store.class);
+            stores.add(store);
+        }
+        return new Cluster(clusterId, stores);
+    }
+
+    @Override
+    public Long getOrCreateStoreId(final long clusterId, final Endpoint endpoint) {
+        final String storeIdKey = MetadataKeyHelper.getStoreIdKey(clusterId, endpoint);
+        final byte[] bytesVal = this.rheaKVStore.bGet(storeIdKey);
+        if (bytesVal == null) {
+            final String storeSeqKey = MetadataKeyHelper.getStoreSeqKey(clusterId);
+            LongSequence storeSequence = this.storeSequenceMap.get(storeSeqKey);
+            if (storeSequence == null) {
+                final LongSequence newStoreSequence = new LongSequence() {
+
+                    @Override
+                    public Sequence getNextSequence() {
+                        return rheaKVStore.bGetSequence(storeSeqKey, 32);
+                    }
+                };
+                storeSequence = this.storeSequenceMap.putIfAbsent(storeSeqKey, newStoreSequence);
+                if (storeSequence == null) {
+                    storeSequence = newStoreSequence;
+                }
+            }
+            final long newStoreId = storeSequence.next();
+            final byte[] newBytesVal = new byte[8];
+            Bits.putLong(newBytesVal, 0, newStoreId);
+            final byte[] oldBytesVal = this.rheaKVStore.bPutIfAbsent(storeIdKey, newBytesVal);
+            if (oldBytesVal != null) {
+                return Bits.getLong(oldBytesVal, 0);
+            } else {
+                return newStoreId;
+            }
+        }
+        return Bits.getLong(bytesVal, 0);
+    }
+
+    @Override
+    public Store getStoreInfo(final long clusterId, final long storeId) {
+        final String storeInfoKey = MetadataKeyHelper.getStoreInfoKey(clusterId, storeId);
+        final byte[] bytes = this.rheaKVStore.bGet(storeInfoKey);
+        if (bytes == null) {
+            Store empty = new Store();
+            empty.setId(storeId);
+            return empty;
+        }
+        return this.serializer.readObject(bytes, Store.class);
+    }
+
+    @Override
+    public Store getStoreInfo(final long clusterId, final Endpoint endpoint) {
+        final long storeId = getOrCreateStoreId(clusterId, endpoint);
+        return getStoreInfo(clusterId, storeId);
+    }
+
+    @Override
+    public CompletableFuture updateStoreInfo(final long clusterId, final Store store) {
+        final long storeId = store.getId();
+        final String storeInfoKey = MetadataKeyHelper.getStoreInfoKey(clusterId, storeId);
+        final byte[] bytes = this.serializer.writeObject(store);
+        final CompletableFuture future = new CompletableFuture<>();
+        this.rheaKVStore.getAndPut(storeInfoKey, bytes).whenComplete((prevBytes, getPutThrowable) -> {
+            if (getPutThrowable == null) {
+                if (prevBytes != null) {
+                    future.complete(serializer.readObject(prevBytes, Store.class));
+                } else {
+                    mergeClusterIndex(clusterId, storeId).whenComplete((ignored, mergeThrowable) -> {
+                        if (mergeThrowable == null) {
+                            future.complete(null);
+                        } else {
+                            future.completeExceptionally(mergeThrowable);
+                        }
+                    });
+                }
+            } else {
+                future.completeExceptionally(getPutThrowable);
+            }
+        });
+        return future;
+    }
+
+    @Override
+    public Long createRegionId(final long clusterId) {
+        final String regionSeqKey = MetadataKeyHelper.getRegionSeqKey(clusterId);
+        LongSequence regionSequence = this.regionSequenceMap.get(regionSeqKey);
+        if (regionSequence == null) {
+            final LongSequence newRegionSequence = new LongSequence(Region.MAX_ID_WITH_MANUAL_CONF) {
+
+                @Override
+                public Sequence getNextSequence() {
+                    return rheaKVStore.bGetSequence(regionSeqKey, 32);
+                }
+            };
+            regionSequence = this.regionSequenceMap.putIfAbsent(regionSeqKey, newRegionSequence);
+            if (regionSequence == null) {
+                regionSequence = newRegionSequence;
+            }
+        }
+        return regionSequence.next();
+    }
+
+    @Override
+    public StoreStats getStoreStats(final long clusterId, final long storeId) {
+        final String key = MetadataKeyHelper.getStoreStatsKey(clusterId, storeId);
+        final byte[] bytes = this.rheaKVStore.bGet(key);
+        if (bytes == null) {
+            return null;
+        }
+        return this.serializer.readObject(bytes, StoreStats.class);
+    }
+
+    @Override
+    public CompletableFuture updateStoreStats(final long clusterId, final StoreStats storeStats) {
+        final String key = MetadataKeyHelper.getStoreStatsKey(clusterId, storeStats.getStoreId());
+        final byte[] bytes = this.serializer.writeObject(storeStats);
+        return this.rheaKVStore.put(key, bytes);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Pair getRegionStats(final long clusterId, final Region region) {
+        final String key = MetadataKeyHelper.getRegionStatsKey(clusterId, region.getId());
+        final byte[] bytes = this.rheaKVStore.bGet(key);
+        if (bytes == null) {
+            return null;
+        }
+        return this.serializer.readObject(bytes, Pair.class);
+    }
+
+    @Override
+    public CompletableFuture updateRegionStats(final long clusterId, final Region region,
+                                                        final RegionStats regionStats) {
+        final String key = MetadataKeyHelper.getRegionStatsKey(clusterId, region.getId());
+        final byte[] bytes = this.serializer.writeObject(Pair.of(region, regionStats));
+        return this.rheaKVStore.put(key, bytes);
+    }
+
+    @Override
+    public CompletableFuture batchUpdateRegionStats(final long clusterId,
+                                                             final List> regionStatsList) {
+        final List entries = Lists.newArrayListWithCapacity(regionStatsList.size());
+        for (final Pair p : regionStatsList) {
+            final String key = MetadataKeyHelper.getRegionStatsKey(clusterId, p.getKey().getId());
+            final byte[] bytes = this.serializer.writeObject(p);
+            entries.add(new KVEntry(BytesUtil.writeUtf8(key), bytes));
+        }
+        return this.rheaKVStore.put(entries);
+    }
+
+    @Override
+    public Set unsafeGetStoreIds(final long clusterId) {
+        Set storeIds = this.clusterStoreIdsCache.get(clusterId);
+        if (storeIds != null) {
+            return storeIds;
+        }
+        storeIds = getClusterIndex(clusterId);
+        this.clusterStoreIdsCache.put(clusterId, storeIds);
+        return storeIds;
+    }
+
+    @Override
+    public Map unsafeGetStoreIdsByEndpoints(final long clusterId, final List endpoints) {
+        if (endpoints == null || endpoints.isEmpty()) {
+            return Collections.emptyMap();
+        }
+        final List storeIdKeyList = Lists.newArrayListWithCapacity(endpoints.size());
+        final Map keyToEndpointMap = Maps.newHashMapWithExpectedSize(endpoints.size());
+        for (final Endpoint endpoint : endpoints) {
+            final byte[] keyBytes = BytesUtil.writeUtf8(MetadataKeyHelper.getStoreIdKey(clusterId, endpoint));
+            storeIdKeyList.add(keyBytes);
+            keyToEndpointMap.put(ByteArray.wrap(keyBytes), endpoint);
+        }
+        final Map storeIdBytes = this.rheaKVStore.bMultiGet(storeIdKeyList);
+        final Map ids = Maps.newHashMapWithExpectedSize(storeIdBytes.size());
+        for (final Map.Entry entry : storeIdBytes.entrySet()) {
+            final Long storeId = Bits.getLong(entry.getValue(), 0);
+            final Endpoint endpoint = keyToEndpointMap.get(entry.getKey());
+            ids.put(storeId, endpoint);
+        }
+        return ids;
+    }
+
+    @Override
+    public void invalidCache() {
+        this.clusterStoreIdsCache.clear();
+    }
+
+    private Set getClusterIndex(final long clusterId) {
+        final String key = MetadataKeyHelper.getClusterInfoKey(clusterId);
+        final byte[] indexBytes = this.rheaKVStore.bGet(key);
+        if (indexBytes == null) {
+            return null;
+        }
+        final String strVal = BytesUtil.readUtf8(indexBytes);
+        final String[] array = Strings.split(strVal, ',');
+        if (array == null) {
+            return null;
+        }
+        final Set storeIds = new HashSet<>(array.length);
+        for (final String str : array) {
+            storeIds.add(Long.parseLong(str.trim()));
+        }
+        return storeIds;
+    }
+
+    private CompletableFuture mergeClusterIndex(final long clusterId, final long storeId) {
+        final String key = MetadataKeyHelper.getClusterInfoKey(clusterId);
+        final CompletableFuture future = this.rheaKVStore.merge(key, String.valueOf(storeId));
+        future.whenComplete((ignored, throwable) -> {
+            if (throwable != null) {
+                LOG.error("Fail to merge cluster index, {}, {}.", key, StackTraceUtil.stackTrace(throwable));
+            }
+            clusterStoreIdsCache.clear();
+        });
+        return future;
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultPlacementDriverService.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultPlacementDriverService.java
new file mode 100644
index 0000000..ae6440b
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/DefaultPlacementDriverService.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatResponse;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+import com.alipay.sofa.jraft.rhea.pipeline.event.RegionPingEvent;
+import com.alipay.sofa.jraft.rhea.pipeline.event.StorePingEvent;
+import com.alipay.sofa.jraft.rhea.pipeline.handler.LogHandler;
+import com.alipay.sofa.jraft.rhea.pipeline.handler.PlacementDriverTailHandler;
+import com.alipay.sofa.jraft.rhea.util.StackTraceUtil;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rhea.util.pipeline.DefaultHandlerInvoker;
+import com.alipay.sofa.jraft.rhea.util.pipeline.DefaultPipeline;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerInvoker;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Pipeline;
+import com.alipay.sofa.jraft.rhea.util.pipeline.future.PipelineFuture;
+import com.alipay.sofa.jraft.util.JRaftServiceLoader;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class DefaultPlacementDriverService implements PlacementDriverService, LeaderStateListener {
+
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultPlacementDriverService.class);
+
+    private final RheaKVStore   rheaKVStore;
+
+    private MetadataStore       metadataStore;
+    private HandlerInvoker      pipelineInvoker;
+    private Pipeline            pipeline;
+    private volatile boolean    isLeader;
+
+    private boolean             started;
+
+    public DefaultPlacementDriverService(RheaKVStore rheaKVStore) {
+        this.rheaKVStore = rheaKVStore;
+    }
+
+    @Override
+    public synchronized boolean init(final PlacementDriverServerOptions opts) {
+        if (this.started) {
+            LOG.info("[DefaultPlacementDriverService] already started.");
+            return true;
+        }
+        Requires.requireNonNull(opts, "placementDriverServerOptions");
+        this.metadataStore = new DefaultMetadataStore(this.rheaKVStore);
+        final ThreadPoolExecutor threadPool = createPipelineExecutor(opts);
+        if (threadPool != null) {
+            this.pipelineInvoker = new DefaultHandlerInvoker(threadPool);
+        }
+        this.pipeline = new DefaultPipeline(); //
+        initPipeline(this.pipeline);
+        LOG.info("[DefaultPlacementDriverService] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        try {
+            if (this.pipelineInvoker != null) {
+                this.pipelineInvoker.shutdown();
+            }
+            invalidLocalCache();
+        } finally {
+            this.started = false;
+            LOG.info("[DefaultPlacementDriverService] shutdown successfully.");
+        }
+    }
+
+    @Override
+    public void handleStoreHeartbeatRequest(final StoreHeartbeatRequest request,
+                                            final RequestProcessClosure closure) {
+        final StoreHeartbeatResponse response = new StoreHeartbeatResponse();
+        response.setClusterId(request.getClusterId());
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            // Only save the data
+            final StorePingEvent storePingEvent = new StorePingEvent(request, this.metadataStore);
+            final PipelineFuture future = this.pipeline.invoke(storePingEvent);
+            future.whenComplete((ignored, throwable) -> {
+                if (throwable != null) {
+                    LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(throwable));
+                    response.setError(Errors.forException(throwable));
+                }
+                closure.sendResponse(response);
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleRegionHeartbeatRequest(final RegionHeartbeatRequest request,
+                                             final RequestProcessClosure closure) {
+        final RegionHeartbeatResponse response = new RegionHeartbeatResponse();
+        response.setClusterId(request.getClusterId());
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            // 1. First, save the data
+            // 2. Second, check if need to send a dispatch instruction
+            final RegionPingEvent regionPingEvent = new RegionPingEvent(request, this.metadataStore);
+            final PipelineFuture> future = this.pipeline.invoke(regionPingEvent);
+            future.whenComplete((instructions, throwable) -> {
+                if (throwable == null) {
+                    response.setValue(instructions);
+                } else {
+                    LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(throwable));
+                    response.setError(Errors.forException(throwable));
+                }
+                closure.sendResponse(response);
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleGetClusterInfoRequest(final GetClusterInfoRequest request,
+                                            final RequestProcessClosure closure) {
+        final long clusterId = request.getClusterId();
+        final GetClusterInfoResponse response = new GetClusterInfoResponse();
+        response.setClusterId(clusterId);
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            final Cluster cluster = this.metadataStore.getClusterInfo(clusterId);
+            response.setValue(cluster);
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+        }
+        closure.sendResponse(response);
+    }
+
+    @Override
+    public void handleGetStoreInfoRequest(final GetStoreInfoRequest request,
+                                          final RequestProcessClosure closure) {
+        final long clusterId = request.getClusterId();
+        final GetStoreInfoResponse response = new GetStoreInfoResponse();
+        response.setClusterId(clusterId);
+        LOG.info("Handling {}.", request);
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            final Store store = this.metadataStore.getStoreInfo(clusterId, request.getEndpoint());
+            response.setValue(store);
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+        }
+        closure.sendResponse(response);
+    }
+
+    @Override
+    public void handleSetStoreInfoRequest(final SetStoreInfoRequest request,
+                                          final RequestProcessClosure closure) {
+        final long clusterId = request.getClusterId();
+        final SetStoreInfoResponse response = new SetStoreInfoResponse();
+        response.setClusterId(clusterId);
+        LOG.info("Handling {}.", request);
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            final CompletableFuture future = this.metadataStore.updateStoreInfo(clusterId, request.getStore());
+            future.whenComplete((prevStore, throwable) -> {
+                if (throwable == null) {
+                    response.setValue(prevStore);
+                } else {
+                    LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(throwable));
+                    response.setError(Errors.forException(throwable));
+                }
+                closure.sendResponse(response);
+            });
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+            closure.sendResponse(response);
+        }
+    }
+
+    @Override
+    public void handleGetStoreIdRequest(final GetStoreIdRequest request,
+                                        final RequestProcessClosure closure) {
+        final long clusterId = request.getClusterId();
+        final GetStoreIdResponse response = new GetStoreIdResponse();
+        response.setClusterId(clusterId);
+        LOG.info("Handling {}.", request);
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            final Long storeId = this.metadataStore.getOrCreateStoreId(clusterId, request.getEndpoint());
+            response.setValue(storeId);
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+        }
+        closure.sendResponse(response);
+    }
+
+    @Override
+    public void handleCreateRegionIdRequest(final CreateRegionIdRequest request,
+                                            final RequestProcessClosure closure) {
+        final long clusterId = request.getClusterId();
+        final CreateRegionIdResponse response = new CreateRegionIdResponse();
+        response.setClusterId(clusterId);
+        LOG.info("Handling {}.", request);
+        if (!this.isLeader) {
+            response.setError(Errors.NOT_LEADER);
+            closure.sendResponse(response);
+            return;
+        }
+        try {
+            final Long newRegionId = this.metadataStore.createRegionId(clusterId);
+            response.setValue(newRegionId);
+        } catch (final Throwable t) {
+            LOG.error("Failed to handle: {}, {}.", request, StackTraceUtil.stackTrace(t));
+            response.setError(Errors.forException(t));
+        }
+        closure.sendResponse(response);
+    }
+
+    @Override
+    public void onLeaderStart(final long leaderTerm) {
+        this.isLeader = true;
+        invalidLocalCache();
+    }
+
+    @Override
+    public void onLeaderStop(final long leaderTerm) {
+        this.isLeader = false;
+        invalidLocalCache();
+    }
+
+    protected void initPipeline(final Pipeline pipeline) {
+        final List sortedHandlers = JRaftServiceLoader.load(Handler.class) //
+            .sort();
+
+        // default handlers and order:
+        //
+        // 1. logHandler
+        // 2. storeStatsValidator
+        // 3. regionStatsValidator
+        // 4. storeStatsPersistence
+        // 5. regionStatsPersistence
+        // 6. regionLeaderBalance
+        // 7. splittingJudgeByApproximateKeys
+        // 8: placementDriverTail
+        for (final Handler h : sortedHandlers) {
+            pipeline.addLast(h);
+        }
+
+        // first handler
+        pipeline.addFirst(this.pipelineInvoker, "logHandler", new LogHandler());
+        // last handler
+        pipeline.addLast("placementDriverTail", new PlacementDriverTailHandler());
+    }
+
+    private void invalidLocalCache() {
+        if (this.metadataStore != null) {
+            this.metadataStore.invalidCache();
+        }
+        ClusterStatsManager.invalidCache();
+    }
+
+    private ThreadPoolExecutor createPipelineExecutor(final PlacementDriverServerOptions opts) {
+        final int corePoolSize = opts.getPipelineCorePoolSize();
+        final int maximumPoolSize = opts.getPipelineMaximumPoolSize();
+        if (corePoolSize <= 0 || maximumPoolSize <= 0) {
+            return null;
+        }
+
+        final String name = "rheakv-pipeline-executor";
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(false) //
+            .coreThreads(corePoolSize) //
+            .maximumThreads(maximumPoolSize) //
+            .keepAliveSeconds(120L) //
+            .workQueue(new ArrayBlockingQueue<>(1024)) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(new CallerRunsPolicyWithReport(name)) //
+            .build();
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataKeyHelper.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataKeyHelper.java
new file mode 100644
index 0000000..cdd1f5b
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataKeyHelper.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.rhea.util.StringBuilderHelper;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ * @author jiachun.fjc
+ */
+public final class MetadataKeyHelper {
+
+    private static final char DELIMITER = '-';
+
+    public static String getClusterInfoKey(final long clusterId) {
+        return StringBuilderHelper.get() //
+            .append("pd_cluster") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .toString();
+    }
+
+    public static String getStoreIdKey(final long clusterId, final Endpoint endpoint) {
+        return StringBuilderHelper.get() //
+            .append("pd_store_id_map") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .append(DELIMITER) //
+            .append(endpoint) //
+            .toString();
+    }
+
+    public static String getStoreSeqKey(final long clusterId) {
+        return StringBuilderHelper.get() //
+            .append("pd_store_id_seq") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .toString();
+    }
+
+    public static String getStoreInfoKey(final long clusterId, final long storeId) {
+        return StringBuilderHelper.get() //
+            .append("pd_store_info") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .append(DELIMITER) //
+            .append(storeId) //
+            .toString();
+    }
+
+    public static String getRegionSeqKey(final long clusterId) {
+        return StringBuilderHelper.get() //
+            .append("pd_region_id_seq") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .toString();
+    }
+
+    public static String getStoreStatsKey(final long clusterId, final long storeId) {
+        return StringBuilderHelper.get() //
+            .append("pd_store_stats") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .append(DELIMITER) //
+            .append(storeId).toString();
+    }
+
+    public static String getRegionStatsKey(final long clusterId, final long regionId) {
+        return StringBuilderHelper.get() //
+            .append("pd_region_stats") //
+            .append(DELIMITER) //
+            .append(clusterId) //
+            .append(DELIMITER) //
+            .append(regionId) //
+            .toString();
+    }
+
+    private MetadataKeyHelper() {
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataStore.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataStore.java
new file mode 100644
index 0000000..cc6b774
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/MetadataStore.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+import com.alipay.sofa.jraft.rhea.metadata.Cluster;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.Store;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.util.Endpoint;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public interface MetadataStore {
+
+    /**
+     * Returns the specified cluster information.
+     */
+    Cluster getClusterInfo(final long clusterId);
+
+    /**
+     * The pd server stores the storeIds of all nodes.
+     * This method provides a lookup for the storeId according
+     * to the endpoint.  If there is no value, then a globally
+     * unique storeId is created.
+     */
+    Long getOrCreateStoreId(final long clusterId, final Endpoint endpoint);
+
+    /**
+     * Query the store information by the storeId.  If the result
+     * is a empty instance, the caller needs to use its own local
+     * configuration.
+     */
+    Store getStoreInfo(final long clusterId, final long storeId);
+
+    /**
+     * Query the store information by the endpoint.  If the result
+     * is a empty instance, the caller needs to use its own local
+     * configuration.
+     */
+    Store getStoreInfo(final long clusterId, final Endpoint endpoint);
+
+    /**
+     * Update the store information by the storeId, and return the
+     * previous value.
+     */
+    CompletableFuture updateStoreInfo(final long clusterId, final Store store);
+
+    /**
+     * Create a globally unique regionId.
+     */
+    Long createRegionId(final long clusterId);
+
+    /**
+     * Returns the stats information of the specified store.
+     */
+    StoreStats getStoreStats(final long clusterId, final long storeId);
+
+    /**
+     * Update the stats information of the specified store.
+     */
+    CompletableFuture updateStoreStats(final long clusterId, final StoreStats storeStats);
+
+    /**
+     * Returns the stats information of the specified region.
+     */
+    Pair getRegionStats(final long clusterId, final Region region);
+
+    /**
+     * Update the stats information of the specified region.
+     */
+    CompletableFuture updateRegionStats(final long clusterId, final Region region,
+                                                 final RegionStats regionStats);
+
+    /**
+     * Batch update the stats information of regions.
+     */
+    CompletableFuture batchUpdateRegionStats(final long clusterId,
+                                                      final List> regionStatsList);
+
+    Set unsafeGetStoreIds(final long clusterId);
+
+    Map unsafeGetStoreIdsByEndpoints(final long clusterId, final List endpoints);
+
+    /**
+     * Clear the cache.
+     */
+    void invalidCache();
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverProcessor.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverProcessor.java
new file mode 100644
index 0000000..6dc8377
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverProcessor.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.concurrent.Executor;
+
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.errors.RheaRuntimeException;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.util.Requires;
+
+/**
+ * The PD server does not actively push information to any RheaKV-node,
+ * and all information about the cluster is reported by the RheaKV-node.
+ *
+ * 
    + * PD server needs to handle two types of heartbeats:
+ *  1. One case is the heartbeat of 'Store'. The PD is processed in the
+ *      'handleStoreHeartbeat' method.  The main task is to save the
+ *      state of current 'Store', including the number of regions, and
+ *      how many region-leader in the 'Store', etc. This information will
+ *      be used as scheduling reference.
+ *
+ *  2. The other case is the heartbeat reported by the region-leader. The
+ *      PD is processed in the 'handleRegionHeartbeat' method. Note that
+ *      only the region-leader can report the 'Region' information, and
+ *      the flower cant not.
+ * 
    
+ *
+ * @author jiachun.fjc
+ */
+public class PlacementDriverProcessor implements RpcProcessor {
+
+    private final Class               reqClazz;
+    private final PlacementDriverService placementDriverService;
+    private final Executor               executor;
+
+    public PlacementDriverProcessor(Class reqClazz, PlacementDriverService placementDriverService, Executor executor) {
+        this.reqClazz = Requires.requireNonNull(reqClazz, "reqClazz");
+        this.placementDriverService = Requires.requireNonNull(placementDriverService, "placementDriverService");
+        this.executor = executor;
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final T request) {
+        Requires.requireNonNull(request, "request");
+        final RequestProcessClosure closure = new RequestProcessClosure<>(request, rpcCtx);
+        switch (request.magic()) {
+            case BaseRequest.STORE_HEARTBEAT:
+                this.placementDriverService.handleStoreHeartbeatRequest((StoreHeartbeatRequest) request, closure);
+                break;
+            case BaseRequest.REGION_HEARTBEAT:
+                this.placementDriverService.handleRegionHeartbeatRequest((RegionHeartbeatRequest) request, closure);
+                break;
+            case BaseRequest.GET_CLUSTER_INFO:
+                this.placementDriverService.handleGetClusterInfoRequest((GetClusterInfoRequest) request, closure);
+                break;
+            case BaseRequest.GET_STORE_INFO:
+                this.placementDriverService.handleGetStoreInfoRequest((GetStoreInfoRequest) request, closure);
+                break;
+            case BaseRequest.SET_STORE_INFO:
+                this.placementDriverService.handleSetStoreInfoRequest((SetStoreInfoRequest) request, closure);
+                break;
+            case BaseRequest.GET_STORE_ID:
+                this.placementDriverService.handleGetStoreIdRequest((GetStoreIdRequest) request, closure);
+                break;
+            case BaseRequest.CREATE_REGION_ID:
+                this.placementDriverService.handleCreateRegionIdRequest((CreateRegionIdRequest) request, closure);
+                break;
+            default:
+                throw new RheaRuntimeException("Unsupported request type: " + request.getClass().getName());
+        }
+    }
+
+    @Override
+    public String interest() {
+        return reqClazz.getName();
+    }
+
+    @Override
+    public Executor executor() {
+        return this.executor;
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverServer.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverServer.java
new file mode 100644
index 0000000..d24d819
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverServer.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rhea.client.DefaultRheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.RheaKVStore;
+import com.alipay.sofa.jraft.rhea.client.pd.PlacementDriverClient;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.util.concurrent.CallerRunsPolicyWithReport;
+import com.alipay.sofa.jraft.rhea.util.concurrent.NamedThreadFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.ExecutorServiceHelper;
+import com.alipay.sofa.jraft.util.Requires;
+import com.alipay.sofa.jraft.util.ThreadPoolUtil;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * PlacementDriverServer is a role responsible for overall global control.
+ *
+ * 
    + *     ┌───────────────────────────────────────────┐
+ *     │ PlacementDriverClient->MetadataRpcClient  │
+ *     └───────────────────────────────────────────┘
+ *                           │
+ *                           │              ┌───────────────────────────────────────────┐
+ *                           │              │       StoreEngine->HeartbeatSender        │
+ *                           │              └───────────────────────────────────────────┘
+ *                           │                                    │
+ *                           │                   ┌────────────────┴─────────┐
+ *                           │                   │                          │
+ *                           │                   ▼                          ▼
+ *                           │       ┌──────────────────────┐   ┌───────────────────────┐
+ *                  ┌────────┘       │  StoreHeartbeatTask  │   │  RegionHeartbeatTask  │──┐
+ *                 rpc               └──────────────────────┘   └───────────────────────┘  │
+ *                  │                        │           ▲                 ▲               │
+ *                  │                        │           │                 │               │
+ *                  │                        │           ▼                 ▼               │
+ *                  │  ┌───────rpc───────────┘     ┌─────────────────────────────┐         │
+ *                  │  │                           │       StatsCollector        │         │
+ *                  │  │                           └─────────────────────────────┘         │
+ *                  │  │                                                                   │
+ *                  ▼  ▼                                                                   │
+ *     ┌──────────────────────────────┐                                                    │
+ *     │    PlacementDriverServer     │◀────────────────────────────rpc────────────────────┘
+ *     └──────────────────────────────┘
+ *                     │
+ *                     │
+ *                     ▼
+ *     ┌──────────────────────────────┐       ┌──────────────────────────────┐
+ *     │   PlacementDriverProcessor   │─────▶ │    PlacementDriverService    │
+ *     └──────────────────────────────┘       └──────────────────────────────┘
+ *                                                            │
+ *                                 ┌──────────────────────────┴─────────────┐
+ *                                 │                                        │
+ *                                 ▼                                        ▼
+ *                 ┌──────────────────────────────┐         ┌──────────────────────────────┐
+ *                 │           Pipeline           │         │        MetadataStore         │
+ *                 └──────────────────────────────┘         └──────────────────────────────┘
+ *                                 │
+ *                                 │
+ *                                 ▼
+ *                 ┌──────────────────────────────┐
+ *                 │         Handlers ...         │
+ *                 └──────────────────────────────┘
+ * 
    
+ *
+ * @author jiachun.fjc
+ */
+public class PlacementDriverServer implements Lifecycle {
+
+    private static final Logger      LOG = LoggerFactory.getLogger(PlacementDriverServer.class);
+
+    private final ThreadPoolExecutor pdExecutor;
+
+    private PlacementDriverService   placementDriverService;
+    private RheaKVStore              rheaKVStore;
+    private RegionEngine             regionEngine;
+
+    private boolean                  started;
+
+    public PlacementDriverServer() {
+        this(null);
+    }
+
+    public PlacementDriverServer(ThreadPoolExecutor pdExecutor) {
+        this.pdExecutor = pdExecutor != null ? pdExecutor : createDefaultPdExecutor();
+    }
+
+    @Override
+    public synchronized boolean init(final PlacementDriverServerOptions opts) {
+        if (this.started) {
+            LOG.info("[PlacementDriverServer] already started.");
+            return true;
+        }
+        Requires.requireNonNull(opts, "opts");
+        final RheaKVStoreOptions rheaOpts = opts.getRheaKVStoreOptions();
+        Requires.requireNonNull(rheaOpts, "opts.rheaKVStoreOptions");
+        this.rheaKVStore = new DefaultRheaKVStore();
+        if (!this.rheaKVStore.init(rheaOpts)) {
+            LOG.error("Fail to init [RheaKVStore].");
+            return false;
+        }
+        this.placementDriverService = new DefaultPlacementDriverService(this.rheaKVStore);
+        if (!this.placementDriverService.init(opts)) {
+            LOG.error("Fail to init [PlacementDriverService].");
+            return false;
+        }
+        final StoreEngine storeEngine = ((DefaultRheaKVStore) this.rheaKVStore).getStoreEngine();
+        Requires.requireNonNull(storeEngine, "storeEngine");
+        final List regionEngines = storeEngine.getAllRegionEngines();
+        if (regionEngines.isEmpty()) {
+            throw new IllegalArgumentException("Non region for [PlacementDriverServer]");
+        }
+        if (regionEngines.size() > 1) {
+            throw new IllegalArgumentException("Only support single region for [PlacementDriverServer]");
+        }
+        this.regionEngine = regionEngines.get(0);
+        this.rheaKVStore.addLeaderStateListener(this.regionEngine.getRegion().getId(),
+            ((DefaultPlacementDriverService) this.placementDriverService));
+        addPlacementDriverProcessor(storeEngine.getRpcServer());
+        LOG.info("[PlacementDriverServer] start successfully, options: {}.", opts);
+        return this.started = true;
+    }
+
+    @Override
+    public synchronized void shutdown() {
+        if (!this.started) {
+            return;
+        }
+        if (this.rheaKVStore != null) {
+            this.rheaKVStore.shutdown();
+        }
+        if (this.placementDriverService != null) {
+            this.placementDriverService.shutdown();
+        }
+        ExecutorServiceHelper.shutdownAndAwaitTermination(this.pdExecutor);
+        this.started = false;
+        LOG.info("[PlacementDriverServer] shutdown successfully.");
+    }
+
+    public ThreadPoolExecutor getPdExecutor() {
+        return pdExecutor;
+    }
+
+    public PlacementDriverService getPlacementDriverService() {
+        return placementDriverService;
+    }
+
+    public RheaKVStore getRheaKVStore() {
+        return rheaKVStore;
+    }
+
+    public RegionEngine getRegionEngine() {
+        return regionEngine;
+    }
+
+    public boolean isLeader() {
+        return this.regionEngine.isLeader();
+    }
+
+    public PeerId getLeaderId() {
+        return this.regionEngine.getLeaderId();
+    }
+
+    public boolean awaitReady(final long timeoutMillis) {
+        final PlacementDriverClient pdClient = this.rheaKVStore.getPlacementDriverClient();
+        final Endpoint endpoint = pdClient.getLeader(this.regionEngine.getRegion().getId(), true, timeoutMillis);
+        return endpoint != null;
+    }
+
+    private void addPlacementDriverProcessor(final RpcServer rpcServer) {
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(RegionHeartbeatRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(StoreHeartbeatRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(GetClusterInfoRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(GetStoreIdRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(GetStoreInfoRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(SetStoreInfoRequest.class,
+            this.placementDriverService, this.pdExecutor));
+        rpcServer.registerProcessor(new PlacementDriverProcessor<>(CreateRegionIdRequest.class,
+            this.placementDriverService, this.pdExecutor));
+    }
+
+    private ThreadPoolExecutor createDefaultPdExecutor() {
+        final int corePoolSize = Math.max(Utils.cpus() << 2, 32);
+        final String name = "rheakv-pd-executor";
+        return ThreadPoolUtil.newBuilder() //
+            .poolName(name) //
+            .enableMetric(true) //
+            .coreThreads(corePoolSize) //
+            .maximumThreads(corePoolSize << 2) //
+            .keepAliveSeconds(120L) //
+            .workQueue(new ArrayBlockingQueue<>(4096)) //
+            .threadFactory(new NamedThreadFactory(name, true)) //
+            .rejectedHandler(new CallerRunsPolicyWithReport(name, name)) //
+            .build();
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverService.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverService.java
new file mode 100644
index 0000000..75c59b2
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverService.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import com.alipay.sofa.jraft.Lifecycle;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.BaseResponse;
+import com.alipay.sofa.jraft.rhea.cmd.pd.CreateRegionIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetClusterInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreIdRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.GetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.SetStoreInfoRequest;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+
+/**
+ * @author jiachun.fjc
+ */
+public interface PlacementDriverService extends Lifecycle {
+
+    /**
+     * {@link BaseRequest#STORE_HEARTBEAT}
+     */
+    void handleStoreHeartbeatRequest(final StoreHeartbeatRequest request,
+                                     final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#REGION_HEARTBEAT}
+     */
+    void handleRegionHeartbeatRequest(final RegionHeartbeatRequest request,
+                                      final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#GET_CLUSTER_INFO}
+     */
+    void handleGetClusterInfoRequest(final GetClusterInfoRequest request,
+                                     final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#GET_STORE_INFO}
+     */
+    void handleGetStoreInfoRequest(final GetStoreInfoRequest request,
+                                   final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#SET_STORE_INFO}
+     */
+    void handleSetStoreInfoRequest(final SetStoreInfoRequest request,
+                                   final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#GET_STORE_ID}
+     */
+    void handleGetStoreIdRequest(final GetStoreIdRequest request,
+                                 final RequestProcessClosure closure);
+
+    /**
+     * {@link BaseRequest#CREATE_REGION_ID}
+     */
+    void handleCreateRegionIdRequest(final CreateRegionIdRequest request,
+                                     final RequestProcessClosure closure);
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverStartup.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverStartup.java
new file mode 100644
index 0000000..a782849
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/PlacementDriverStartup.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.errors.PlacementDriverServerStartupException;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class PlacementDriverStartup {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PlacementDriverStartup.class);
+
+    public static void main(String[] args) throws Exception {
+        if (args.length != 1) {
+            LOG.error("Usage: com.alipay.sofa.jraft.rhea.PlacementDriverStartup ");
+            System.exit(1);
+        }
+        final String configPath = args[0];
+        final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+        final PlacementDriverServerOptions opts = mapper.readValue(new File(configPath),
+            PlacementDriverServerOptions.class);
+        final PlacementDriverServer pdServer = new PlacementDriverServer();
+        if (!pdServer.init(opts)) {
+            throw new PlacementDriverServerStartupException("Fail to start [PlacementDriverServer].");
+        }
+        LOG.info("Starting PlacementDriverServer with config: {}.", opts);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/errors/PlacementDriverServerStartupException.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/errors/PlacementDriverServerStartupException.java
new file mode 100644
index 0000000..7026c84
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/errors/PlacementDriverServerStartupException.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.errors;
+
+/**
+ * @author jiachun.fjc
+ */
+public class PlacementDriverServerStartupException extends RuntimeException {
+
+    private static final long serialVersionUID = -7077475444751119247L;
+
+    public PlacementDriverServerStartupException() {
+    }
+
+    public PlacementDriverServerStartupException(String message) {
+        super(message);
+    }
+
+    public PlacementDriverServerStartupException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public PlacementDriverServerStartupException(Throwable cause) {
+        super(cause);
+    }
+
+    public PlacementDriverServerStartupException(String message, Throwable cause, boolean enableSuppression,
+                                                 boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverServerOptions.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverServerOptions.java
new file mode 100644
index 0000000..f57f7d1
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/PlacementDriverServerOptions.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options;
+
+/**
+ * @author jiachun.fjc
+ */
+public class PlacementDriverServerOptions {
+
+    private RheaKVStoreOptions rheaKVStoreOptions;
+    private int                pipelineCorePoolSize;
+    private int                pipelineMaximumPoolSize;
+
+    public RheaKVStoreOptions getRheaKVStoreOptions() {
+        return rheaKVStoreOptions;
+    }
+
+    public void setRheaKVStoreOptions(RheaKVStoreOptions rheaKVStoreOptions) {
+        this.rheaKVStoreOptions = rheaKVStoreOptions;
+    }
+
+    public int getPipelineCorePoolSize() {
+        return pipelineCorePoolSize;
+    }
+
+    public void setPipelineCorePoolSize(int pipelineCorePoolSize) {
+        this.pipelineCorePoolSize = pipelineCorePoolSize;
+    }
+
+    public int getPipelineMaximumPoolSize() {
+        return pipelineMaximumPoolSize;
+    }
+
+    public void setPipelineMaximumPoolSize(int pipelineMaximumPoolSize) {
+        this.pipelineMaximumPoolSize = pipelineMaximumPoolSize;
+    }
+
+    @Override
+    public String toString() {
+        return "PlacementDriverServerOptions{" + "rheaKVStoreOptions=" + rheaKVStoreOptions + ", pipelineCorePoolSize="
+               + pipelineCorePoolSize + ", pipelineMaximumPoolSize=" + pipelineMaximumPoolSize + '}';
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverServerOptionsConfigured.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverServerOptionsConfigured.java
new file mode 100644
index 0000000..92f39d2
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/options/configured/PlacementDriverServerOptionsConfigured.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.options.configured;
+
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+import com.alipay.sofa.jraft.rhea.options.RheaKVStoreOptions;
+import com.alipay.sofa.jraft.rhea.util.Configured;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public final class PlacementDriverServerOptionsConfigured implements Configured {
+
+    private final PlacementDriverServerOptions opts;
+
+    public static PlacementDriverServerOptionsConfigured newConfigured() {
+        return new PlacementDriverServerOptionsConfigured(new PlacementDriverServerOptions());
+    }
+
+    public PlacementDriverServerOptionsConfigured withRheaKVStoreOptions(final RheaKVStoreOptions rheaKVStoreOptions) {
+        this.opts.setRheaKVStoreOptions(rheaKVStoreOptions);
+        return this;
+    }
+
+    public PlacementDriverServerOptionsConfigured withPipelineCorePoolSize(final int pipelineCorePoolSize) {
+        this.opts.setPipelineCorePoolSize(pipelineCorePoolSize);
+        return this;
+    }
+
+    public PlacementDriverServerOptionsConfigured withPipelineMaximumPoolSize(final int pipelineMaximumPoolSize) {
+        this.opts.setPipelineMaximumPoolSize(pipelineMaximumPoolSize);
+        return this;
+    }
+
+    @Override
+    public PlacementDriverServerOptions config() {
+        return this.opts;
+    }
+
+    private PlacementDriverServerOptionsConfigured(PlacementDriverServerOptions opts) {
+        this.opts = opts;
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PingEvent.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PingEvent.java
new file mode 100644
index 0000000..11803c0
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PingEvent.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.event;
+
+import java.util.Collection;
+import java.util.concurrent.LinkedBlockingDeque;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+
+/**
+ * @author jiachun.fjc
+ */
+public abstract class PingEvent extends InboundMessageEvent {
+
+    private final Collection instructions = new LinkedBlockingDeque<>();
+    private final MetadataStore           metadataStore;
+
+    public PingEvent(T message, MetadataStore metadataStore) {
+        super(message);
+        this.metadataStore = metadataStore;
+    }
+
+    public MetadataStore getMetadataStore() {
+        return metadataStore;
+    }
+
+    public Collection getInstructions() {
+        return instructions;
+    }
+
+    public void addInstruction(Instruction instruction) {
+        this.instructions.add(instruction);
+    }
+
+    public boolean isReady() {
+        return !this.instructions.isEmpty();
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PongEvent.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PongEvent.java
new file mode 100644
index 0000000..acd9347
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/PongEvent.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.event;
+
+import java.util.List;
+
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.OutboundMessageEvent;
+
+/**
+ * @author jiachun.fjc
+ */
+public class PongEvent extends OutboundMessageEvent> {
+
+    public PongEvent(long invokeId, List message) {
+        super(invokeId, message);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/RegionPingEvent.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/RegionPingEvent.java
new file mode 100644
index 0000000..12e25a3
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/RegionPingEvent.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.event;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+
+/**
+ * @author jiachun.fjc
+ */
+public class RegionPingEvent extends PingEvent {
+
+    public RegionPingEvent(RegionHeartbeatRequest message, MetadataStore metadataStore) {
+        super(message, metadataStore);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/StorePingEvent.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/StorePingEvent.java
new file mode 100644
index 0000000..a2793ea
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/event/StorePingEvent.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.event;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+public class StorePingEvent extends PingEvent {
+
+    public StorePingEvent(StoreHeartbeatRequest message, MetadataStore metadataStore) {
+        super(message, metadataStore);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/LogHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/LogHandler.java
new file mode 100644
index 0000000..1945559
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/LogHandler.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.pipeline.event.PingEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+
+/**
+ * @author jiachun.fjc
+ */
+@Handler.Sharable
+public class LogHandler extends InboundHandlerAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(LogHandler.class);
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final PingEvent event) throws Exception {
+        LOG.info("New [PingEvent]: {}.", event);
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/PlacementDriverTailHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/PlacementDriverTailHandler.java
new file mode 100644
index 0000000..f7c3e09
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/PlacementDriverTailHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import com.alipay.sofa.jraft.rhea.pipeline.event.PingEvent;
+import com.alipay.sofa.jraft.rhea.pipeline.event.PongEvent;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.rhea.util.pipeline.event.InboundMessageEvent;
+
+/**
+ * @author jiachun.fjc
+ */
+@Handler.Sharable
+public class PlacementDriverTailHandler extends InboundHandlerAdapter> {
+
+    @Override
+    public void handleInbound(final HandlerContext ctx, final InboundMessageEvent event) throws Exception {
+        if (isAcceptable(event)) {
+            // to outbound
+            PingEvent ping = (PingEvent) event;
+            ctx.pipeline().fireOutbound(new PongEvent(ping.getInvokeId(), Lists.newArrayList(ping.getInstructions())));
+        }
+    }
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final PingEvent event) throws Exception {
+        // no-op
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionLeaderBalanceHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionLeaderBalanceHandler.java
new file mode 100644
index 0000000..8a95c9a
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionLeaderBalanceHandler.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.ClusterStatsManager;
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.metadata.Peer;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.pipeline.event.RegionPingEvent;
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.Endpoint;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ * Trying to balance the number of leaders in each store.
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "regionLeaderBalance", priority = 60)
+@Handler.Sharable
+public class RegionLeaderBalanceHandler extends InboundHandlerAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RegionLeaderBalanceHandler.class);
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
+        if (event.isReady()) {
+            return;
+        }
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final RegionHeartbeatRequest request = event.getMessage();
+        final long clusterId = request.getClusterId();
+        final long storeId = request.getStoreId();
+        final ClusterStatsManager clusterStatsManager = ClusterStatsManager.getInstance(clusterId);
+        final List> regionStatsList = request.getRegionStatsList();
+        for (final Pair stats : regionStatsList) {
+            final Region region = stats.getKey();
+            clusterStatsManager.addOrUpdateLeader(storeId, region.getId());
+        }
+
+        // check if the modelWorker
+        final Pair, Integer> modelWorkers = clusterStatsManager.findModelWorkerStores(1);
+        final Set modelWorkerStoreIds = modelWorkers.getKey();
+        final int modelWorkerLeaders = modelWorkers.getValue();
+        if (!modelWorkerStoreIds.contains(storeId)) {
+            return;
+        }
+
+        LOG.info("[Cluster: {}] model worker stores is: {}, it has {} leaders.", clusterId, modelWorkerStoreIds, modelWorkerLeaders);
+
+        for (final Pair pair : regionStatsList) {
+            final Region region = pair.getKey();
+            final List peers = region.getPeers();
+            if (peers == null) {
+                continue;
+            }
+            final List endpoints = Lists.transform(peers, Peer::getEndpoint);
+            final Map storeIds = metadataStore.unsafeGetStoreIdsByEndpoints(clusterId, endpoints);
+            // find lazyWorkers
+            final List> lazyWorkers = clusterStatsManager.findLazyWorkerStores(storeIds.keySet());
+            if (lazyWorkers.isEmpty()) {
+                return;
+            }
+            for (int i = lazyWorkers.size() - 1; i >= 0; i--) {
+                final Pair worker = lazyWorkers.get(i);
+                if (modelWorkerLeaders - worker.getValue() <= 1) { // no need to transfer
+                    lazyWorkers.remove(i);
+                }
+            }
+            if (lazyWorkers.isEmpty()) {
+                continue;
+            }
+            final Pair laziestWorker = tryToFindLaziestWorker(clusterId, metadataStore, lazyWorkers);
+            if (laziestWorker == null) {
+                continue;
+            }
+            final Long lazyWorkerStoreId = laziestWorker.getKey();
+            LOG.info("[Cluster: {}], lazy worker store is: {}, it has {} leaders.", clusterId, lazyWorkerStoreId,
+                    laziestWorker.getValue());
+            final Instruction.TransferLeader transferLeader = new Instruction.TransferLeader();
+            transferLeader.setMoveToStoreId(lazyWorkerStoreId);
+            transferLeader.setMoveToEndpoint(storeIds.get(lazyWorkerStoreId));
+            final Instruction instruction = new Instruction();
+            instruction.setRegion(region.copy());
+            instruction.setTransferLeader(transferLeader);
+            event.addInstruction(instruction);
+            LOG.info("[Cluster: {}], send 'instruction.transferLeader': {} to region: {}.", clusterId, instruction, region);
+            break; // Only do one thing at a time
+        }
+    }
+
+    private Pair tryToFindLaziestWorker(final long clusterId, final MetadataStore metadataStore,
+                                                       final List> lazyWorkers) {
+        final List, StoreStats>> storeStatsList = Lists.newArrayList();
+        for (final Pair worker : lazyWorkers) {
+            final StoreStats stats = metadataStore.getStoreStats(clusterId, worker.getKey());
+            if (stats != null) {
+                // TODO check timeInterval
+                storeStatsList.add(Pair.of(worker, stats));
+            }
+        }
+        if (storeStatsList.isEmpty()) {
+            return null;
+        }
+        if (storeStatsList.size() == 1) {
+            return storeStatsList.get(0).getKey();
+        }
+        final Pair, StoreStats> min = Collections.min(storeStatsList, (o1, o2) -> {
+            final StoreStats s1 = o1.getValue();
+            final StoreStats s2 = o2.getValue();
+            int val = Boolean.compare(s1.isBusy(), s2.isBusy());
+            if (val != 0) {
+                return val;
+            }
+            val = Integer.compare(s1.getRegionCount(), s2.getRegionCount());
+            if (val != 0) {
+                return val;
+            }
+            val = Long.compare(s1.getBytesWritten(), s2.getBytesWritten());
+            if (val != 0) {
+                return val;
+            }
+            val = Long.compare(s1.getBytesRead(), s2.getBytesRead());
+            if (val != 0) {
+                return val;
+            }
+            val = Long.compare(s1.getKeysWritten(), s2.getKeysWritten());
+            if (val != 0) {
+                return val;
+            }
+            val = Long.compare(s1.getKeysRead(), s2.getKeysRead());
+            if (val != 0) {
+                return val;
+            }
+            return Long.compare(-s1.getAvailable(), -s2.getAvailable());
+        });
+        return min.getKey();
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsPersistenceHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsPersistenceHandler.java
new file mode 100644
index 0000000..31dd77c
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsPersistenceHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.pipeline.event.RegionPingEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "regionStatsPersistence", priority = 70)
+@Handler.Sharable
+public class RegionStatsPersistenceHandler extends InboundHandlerAdapter {
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final RegionHeartbeatRequest request = event.getMessage();
+        metadataStore.batchUpdateRegionStats(request.getClusterId(), request.getRegionStatsList()).get(); // sync
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsValidator.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsValidator.java
new file mode 100644
index 0000000..45591be
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/RegionStatsValidator.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionEpoch;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.metadata.TimeInterval;
+import com.alipay.sofa.jraft.rhea.pipeline.event.RegionPingEvent;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "regionStatsValidator", priority = 90)
+@Handler.Sharable
+public class RegionStatsValidator extends InboundHandlerAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(RegionStatsValidator.class);
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final RegionHeartbeatRequest request = event.getMessage();
+        final List> regionStatsList = request.getRegionStatsList();
+        if (regionStatsList == null || regionStatsList.isEmpty()) {
+            LOG.error("Empty [RegionStatsList] by event: {}.", event);
+            throw Errors.INVALID_REGION_STATS.exception();
+        }
+        for (final Pair pair : regionStatsList) {
+            final Region region = pair.getKey();
+            if (region == null) {
+                LOG.error("Empty [Region] by event: {}.", event);
+                throw Errors.INVALID_REGION_STATS.exception();
+            }
+            final RegionEpoch regionEpoch = region.getRegionEpoch();
+            if (regionEpoch == null) {
+                LOG.error("Empty [RegionEpoch] by event: {}.", event);
+                throw Errors.INVALID_REGION_STATS.exception();
+            }
+            final RegionStats regionStats = pair.getValue();
+            if (regionStats == null) {
+                LOG.error("Empty [RegionStats] by event: {}.", event);
+                throw Errors.INVALID_REGION_STATS.exception();
+            }
+            final Pair currentRegionInfo = metadataStore.getRegionStats(request.getClusterId(),
+                region);
+            if (currentRegionInfo == null) {
+                return; // new data
+            }
+            final Region currentRegion = currentRegionInfo.getKey();
+            if (regionEpoch.compareTo(currentRegion.getRegionEpoch()) < 0) {
+                LOG.error("The region epoch is out of date: {}.", event);
+                throw Errors.REGION_HEARTBEAT_OUT_OF_DATE.exception();
+            }
+            final TimeInterval interval = regionStats.getInterval();
+            if (interval == null) {
+                LOG.error("Empty [TimeInterval] by event: {}.", event);
+                throw Errors.INVALID_REGION_STATS.exception();
+            }
+            final TimeInterval currentInterval = currentRegionInfo.getValue().getInterval();
+            if (interval.getEndTimestamp() < currentInterval.getEndTimestamp()) {
+                LOG.error("The [TimeInterval] is out of date: {}.", event);
+                throw Errors.REGION_HEARTBEAT_OUT_OF_DATE.exception();
+            }
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/SplittingJudgeByApproximateKeysHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/SplittingJudgeByApproximateKeysHandler.java
new file mode 100644
index 0000000..6166512
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/SplittingJudgeByApproximateKeysHandler.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import java.util.List;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.ClusterStatsManager;
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.RegionHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.metadata.Instruction;
+import com.alipay.sofa.jraft.rhea.metadata.Region;
+import com.alipay.sofa.jraft.rhea.metadata.RegionStats;
+import com.alipay.sofa.jraft.rhea.pipeline.event.RegionPingEvent;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ * Range split judge, the reference indicator for splitting is the
+ * region's approximate keys.
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "splittingJudgeByApproximateKeys", priority = 50)
+@Handler.Sharable
+public class SplittingJudgeByApproximateKeysHandler extends InboundHandlerAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SplittingJudgeByApproximateKeysHandler.class);
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final RegionPingEvent event) throws Exception {
+        if (event.isReady()) {
+            return;
+        }
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final RegionHeartbeatRequest request = event.getMessage();
+        final long clusterId = request.getClusterId();
+        final ClusterStatsManager clusterStatsManager = ClusterStatsManager.getInstance(clusterId);
+        clusterStatsManager.addOrUpdateRegionStats(request.getRegionStatsList());
+        final Set stores = metadataStore.unsafeGetStoreIds(clusterId);
+        if (stores == null || stores.isEmpty()) {
+            return;
+        }
+        if (clusterStatsManager.regionSize() >= stores.size()) {
+            // one store one region is perfect
+            return;
+        }
+        final Pair modelWorker = clusterStatsManager.findModelWorkerRegion();
+        if (!isSplitNeeded(request, modelWorker)) {
+            return;
+        }
+
+        LOG.info("[Cluster: {}] model worker region is: {}.", clusterId, modelWorker);
+
+        final Long newRegionId = metadataStore.createRegionId(clusterId);
+        final Instruction.RangeSplit rangeSplit = new Instruction.RangeSplit();
+        rangeSplit.setNewRegionId(newRegionId);
+        final Instruction instruction = new Instruction();
+        instruction.setRegion(modelWorker.getKey().copy());
+        instruction.setRangeSplit(rangeSplit);
+        event.addInstruction(instruction);
+    }
+
+    private boolean isSplitNeeded(final RegionHeartbeatRequest request, final Pair modelWorker) {
+        if (modelWorker == null) {
+            return false;
+        }
+        final long modelApproximateKeys = modelWorker.getValue().getApproximateKeys();
+        if (request.getLeastKeysOnSplit() > modelApproximateKeys) {
+            return false;
+        }
+        final Region modelRegion = modelWorker.getKey();
+        final List> regionStatsList = request.getRegionStatsList();
+        for (final Pair p : regionStatsList) {
+            if (modelRegion.equals(p.getKey())) {
+                return true;
+            }
+        }
+        return false;
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsPersistenceHandler.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsPersistenceHandler.java
new file mode 100644
index 0000000..fd00d74
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsPersistenceHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.pipeline.event.StorePingEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "storeStatsPersistence", priority = 80)
+@Handler.Sharable
+public class StoreStatsPersistenceHandler extends InboundHandlerAdapter {
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final StorePingEvent event) throws Exception {
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final StoreHeartbeatRequest request = event.getMessage();
+        metadataStore.updateStoreStats(request.getClusterId(), request.getStats()).get(); // sync
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsValidator.java b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsValidator.java
new file mode 100644
index 0000000..b3a0e3f
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/java/com/alipay/sofa/jraft/rhea/pipeline/handler/StoreStatsValidator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea.pipeline.handler;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.rhea.MetadataStore;
+import com.alipay.sofa.jraft.rhea.cmd.pd.StoreHeartbeatRequest;
+import com.alipay.sofa.jraft.rhea.errors.Errors;
+import com.alipay.sofa.jraft.rhea.metadata.StoreStats;
+import com.alipay.sofa.jraft.rhea.metadata.TimeInterval;
+import com.alipay.sofa.jraft.rhea.pipeline.event.StorePingEvent;
+import com.alipay.sofa.jraft.rhea.util.pipeline.Handler;
+import com.alipay.sofa.jraft.rhea.util.pipeline.HandlerContext;
+import com.alipay.sofa.jraft.rhea.util.pipeline.InboundHandlerAdapter;
+import com.alipay.sofa.jraft.util.SPI;
+
+/**
+ *
+ * @author jiachun.fjc
+ */
+@SPI(name = "storeStatsValidator", priority = 100)
+@Handler.Sharable
+public class StoreStatsValidator extends InboundHandlerAdapter {
+
+    private static final Logger LOG = LoggerFactory.getLogger(StoreStatsValidator.class);
+
+    @Override
+    public void readMessage(final HandlerContext ctx, final StorePingEvent event) throws Exception {
+        final MetadataStore metadataStore = event.getMetadataStore();
+        final StoreHeartbeatRequest request = event.getMessage();
+        final StoreStats storeStats = request.getStats();
+        if (storeStats == null) {
+            LOG.error("Empty [StoreStats] by event: {}.", event);
+            throw Errors.INVALID_STORE_STATS.exception();
+        }
+        final StoreStats currentStoreStats = metadataStore.getStoreStats(request.getClusterId(),
+            storeStats.getStoreId());
+        if (currentStoreStats == null) {
+            return; // new data
+        }
+        final TimeInterval interval = storeStats.getInterval();
+        if (interval == null) {
+            LOG.error("Empty [TimeInterval] by event: {}.", event);
+            throw Errors.INVALID_STORE_STATS.exception();
+        }
+        final TimeInterval currentInterval = currentStoreStats.getInterval();
+        if (interval.getEndTimestamp() < currentInterval.getEndTimestamp()) {
+            LOG.error("The [TimeInterval] is out of date: {}.", event);
+            throw Errors.STORE_HEARTBEAT_OUT_OF_DATE.exception();
+        }
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rhea.util.pipeline.Handler b/jraft-rheakv/rheakv-pd/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rhea.util.pipeline.Handler
new file mode 100644
index 0000000..d61ba30
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/main/resources/META-INF/services/com.alipay.sofa.jraft.rhea.util.pipeline.Handler
@@ -0,0 +1,6 @@
+com.alipay.sofa.jraft.rhea.pipeline.handler.StoreStatsValidator
+com.alipay.sofa.jraft.rhea.pipeline.handler.RegionStatsValidator
+com.alipay.sofa.jraft.rhea.pipeline.handler.StoreStatsPersistenceHandler
+com.alipay.sofa.jraft.rhea.pipeline.handler.RegionStatsPersistenceHandler
+com.alipay.sofa.jraft.rhea.pipeline.handler.RegionLeaderBalanceHandler
+com.alipay.sofa.jraft.rhea.pipeline.handler.SplittingJudgeByApproximateKeysHandler
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/BasePdServer.java b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/BasePdServer.java
new file mode 100644
index 0000000..cce4c56
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/BasePdServer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.concurrent.CopyOnWriteArrayList;
+
+import org.apache.commons.io.FileUtils;
+
+import com.alipay.sofa.jraft.rhea.errors.NotLeaderException;
+import com.alipay.sofa.jraft.rhea.options.PlacementDriverServerOptions;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
+/**
+ * @author jiachun.fjc
+ */
+public class BasePdServer {
+
+    private static final String[]                       CONF         = { "/pd/pd_1.yaml", "/pd/pd_2.yaml",
+            "/pd/pd_3.yaml"                                         };
+
+    private volatile String                             tempDbPath;
+    private volatile String                             tempRaftPath;
+    private CopyOnWriteArrayList pdServerList = new CopyOnWriteArrayList<>();
+
+    protected void start() throws IOException, InterruptedException {
+        System.out.println("PlacementDriverServer init ...");
+        File file = new File("pd_db");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("pd_db");
+        if (file.mkdir()) {
+            this.tempDbPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempDbPath);
+        }
+        file = new File("pd_raft");
+        if (file.exists()) {
+            FileUtils.forceDelete(file);
+        }
+        file = new File("pd_raft");
+        if (file.mkdir()) {
+            this.tempRaftPath = file.getAbsolutePath();
+            System.out.println("make dir: " + this.tempRaftPath);
+        }
+        for (final String c : CONF) {
+            final ObjectMapper mapper = new ObjectMapper(new YAMLFactory());
+            final InputStream in = BasePdServer.class.getResourceAsStream(c);
+            final PlacementDriverServerOptions opts = mapper.readValue(in, PlacementDriverServerOptions.class);
+            final PlacementDriverServer pdServer = new PlacementDriverServer();
+            if (pdServer.init(opts)) {
+                pdServerList.add(pdServer);
+            } else {
+                System.err.println("Fail to init [PlacementDriverServer] witch conf: " + c);
+            }
+        }
+        pdServerList.get(0).awaitReady(10000);
+        System.out.println("Pd server is ready");
+    }
+
+    protected void shutdown() throws IOException {
+        System.out.println("PlacementDriverServer shutdown ...");
+        for (final PlacementDriverServer server : this.pdServerList) {
+            server.shutdown();
+        }
+        if (this.tempDbPath != null) {
+            System.out.println("removing dir: " + this.tempDbPath);
+            FileUtils.forceDelete(new File(this.tempDbPath));
+        }
+        if (this.tempRaftPath != null) {
+            System.out.println("removing dir: " + this.tempRaftPath);
+            FileUtils.forceDelete(new File(this.tempRaftPath));
+        }
+        System.out.println("PlacementDriverServer shutdown complete");
+    }
+
+    protected PlacementDriverServer getLeaderServer() {
+        for (int i = 0; i < 20; i++) {
+            for (final PlacementDriverServer server : this.pdServerList) {
+                if (server.isLeader()) {
+                    return server;
+                }
+            }
+            System.out.println("fail to find leader, try again");
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException ignored) {
+                // ignored
+            }
+        }
+        throw new NotLeaderException("no leader");
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/ClusterStatsManagerTest.java b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/ClusterStatsManagerTest.java
new file mode 100644
index 0000000..3e2359b
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/ClusterStatsManagerTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.alipay.sofa.jraft.rhea.util.Lists;
+import com.alipay.sofa.jraft.rhea.util.Pair;
+
+/**
+ * @author jiachun.fjc
+ */
+public class ClusterStatsManagerTest {
+
+    @Before
+    public void reset() {
+        ClusterStatsManager.invalidCache();
+    }
+
+    @Test
+    public void findModelWorkerStoresTest() {
+        final ClusterStatsManager manager = ClusterStatsManager.getInstance(1);
+        manager.addOrUpdateLeader(10, 101);
+        manager.addOrUpdateLeader(10, 102);
+        manager.addOrUpdateLeader(10, 103);
+        manager.addOrUpdateLeader(11, 104);
+        manager.addOrUpdateLeader(11, 105);
+        manager.addOrUpdateLeader(12, 106);
+        Pair, Integer> result = manager.findModelWorkerStores(1);
+        Assert.assertNotNull(result);
+        Assert.assertEquals(1, result.getKey().size());
+        Assert.assertTrue(result.getKey().contains(10L));
+        Assert.assertEquals(Integer.valueOf(3), result.getValue());
+
+        manager.addOrUpdateLeader(11, 107);
+        result = manager.findModelWorkerStores(1);
+        Assert.assertNotNull(result);
+        Assert.assertTrue(result.getKey().contains(10L));
+        Assert.assertTrue(result.getKey().contains(11L));
+        Assert.assertEquals(2, result.getKey().size());
+        Assert.assertEquals(Integer.valueOf(3), result.getValue());
+
+        manager.addOrUpdateLeader(11, 101);
+        result = manager.findModelWorkerStores(2);
+        Assert.assertNotNull(result);
+        Assert.assertEquals(1, result.getKey().size());
+        Assert.assertEquals(Integer.valueOf(4), result.getValue());
+    }
+
+    @Test
+    public void findLazyWorkerStoresTest() {
+        final ClusterStatsManager manager = ClusterStatsManager.getInstance(1);
+        manager.addOrUpdateLeader(10, 101);
+        manager.addOrUpdateLeader(10, 102);
+        manager.addOrUpdateLeader(10, 103);
+        manager.addOrUpdateLeader(11, 104);
+        manager.addOrUpdateLeader(11, 105);
+        manager.addOrUpdateLeader(12, 106);
+        final Collection storeCandidates = Lists.newArrayList();
+        storeCandidates.add(10L);
+        storeCandidates.add(11L);
+        storeCandidates.add(12L);
+        List> result = manager.findLazyWorkerStores(storeCandidates);
+        Assert.assertNotNull(result);
+        Assert.assertEquals(1, result.size());
+        Assert.assertEquals(Long.valueOf(12), result.get(0).getKey());
+        Assert.assertEquals(Integer.valueOf(1), result.get(0).getValue());
+
+        manager.addOrUpdateLeader(10, 105);
+        result = manager.findLazyWorkerStores(storeCandidates);
+        Assert.assertNotNull(result);
+        Assert.assertEquals(2, result.size());
+        Assert.assertEquals(Integer.valueOf(1), result.get(0).getValue());
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/PdServer.java b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/PdServer.java
new file mode 100644
index 0000000..f3bd51b
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/java/com/alipay/sofa/jraft/rhea/PdServer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.rhea;
+
+import java.io.IOException;
+
+/**
+ * @author jiachun.fjc
+ */
+public class PdServer extends BasePdServer {
+
+    public static void main(String[] args) throws Exception {
+        final PdServer server = new PdServer();
+        server.start();
+
+        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+            try {
+                server.shutdown();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }));
+    }
+}
diff --git a/jraft-rheakv/rheakv-pd/src/test/resources/log4j2.xml b/jraft-rheakv/rheakv-pd/src/test/resources/log4j2.xml
new file mode 100644
index 0000000..25b3a30
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+
+    
+        
+            
+        
+
+        
+            
+            
+                
+            
+            
+        
+
+    
+    
+        
+            
+            
+        
+    
+
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_1.yaml b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_1.yaml
new file mode 100644
index 0000000..44f8565
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_1.yaml
@@ -0,0 +1,17 @@
+##PlacementDriverServerOptions
+---
+rheaKVStoreOptions:
+  clusterName: pd_test
+
+  placementDriverOptions:
+    fake: true
+
+  storeEngineOptions:
+    rocksDBOptions:
+      dbPath: pd_db/
+    raftDataPath: pd_raft/
+    serverAddress:
+      ip: 127.0.0.1
+      port: 8180
+
+  initialServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_2.yaml b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_2.yaml
new file mode 100644
index 0000000..f3fc2ca
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_2.yaml
@@ -0,0 +1,17 @@
+##PlacementDriverServerOptions
+---
+rheaKVStoreOptions:
+  clusterName: pd_test
+
+  placementDriverOptions:
+    fake: true
+
+  storeEngineOptions:
+    rocksDBOptions:
+      dbPath: pd_db/
+    raftDataPath: pd_raft/
+    serverAddress:
+      ip: 127.0.0.1
+      port: 8181
+
+  initialServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
\ No newline at end of file
diff --git a/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_3.yaml b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_3.yaml
new file mode 100644
index 0000000..736edef
--- /dev/null
+++ b/jraft-rheakv/rheakv-pd/src/test/resources/pd/pd_3.yaml
@@ -0,0 +1,17 @@
+##PlacementDriverServerOptions
+---
+rheaKVStoreOptions:
+  clusterName: pd_test
+
+  placementDriverOptions:
+    fake: true
+
+  storeEngineOptions:
+    rocksDBOptions:
+      dbPath: pd_db/
+    raftDataPath: pd_raft/
+    serverAddress:
+      ip: 127.0.0.1
+      port: 8182
+
+  initialServerList: 127.0.0.1:8180,127.0.0.1:8181,127.0.0.1:8182
\ No newline at end of file
diff --git a/jraft-test/config/server.properties b/jraft-test/config/server.properties
new file mode 100644
index 0000000..40ec9be
--- /dev/null
+++ b/jraft-test/config/server.properties
@@ -0,0 +1,5 @@
+groupId=atomic
+serverAddress=127.0.0.1:8611
+conf=127.0.0.1:8609,127.0.0.1:8610,127.0.0.1:8611
+dataPath=./data3/
+totalSlots=1
\ No newline at end of file
diff --git a/jraft-test/pom.xml b/jraft-test/pom.xml
new file mode 100644
index 0000000..a80a75e
--- /dev/null
+++ b/jraft-test/pom.xml
@@ -0,0 +1,48 @@
+
+
+
+    4.0.0
+    
+        jraft-parent
+        com.alipay.sofa
+        1.3.10.bugfix
+    
+    jraft-test
+    jar
+    jraft-test ${project.version}
+
+    
+        
+            ${project.groupId}
+            jraft-core
+        
+        
+            com.alipay.sofa
+            hessian
+        
+        
+            org.slf4j
+            slf4j-api
+        
+        
+            org.apache.logging.log4j
+            log4j-api
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-core
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-slf4j-impl
+            ${log4j.version}
+        
+        
+            org.apache.logging.log4j
+            log4j-jcl
+            ${log4j.version}
+        
+    
+
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashAlgorithm.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashAlgorithm.java
new file mode 100644
index 0000000..beee0c6
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashAlgorithm.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic;
+
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.zip.CRC32;
+
+/**
+ * Hash algorithm
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2017-Nov-22 5:00:08 PM
+ */
+public enum HashAlgorithm {
+    /**
+     * CRC32_HASH as used by the perl API. This will be more consistent both
+     * across multiple API users as well as java versions, but is mostly likely
+     * significantly slower.
+     */
+    CRC32_HASH,
+    /**
+     * FNV hashes are designed to be fast while maintaining a low collision
+     * rate. The FNV speed allows one to quickly hash lots of data while
+     * maintaining a reasonable collision rate.
+     * 
+     * @see 
+     * @see 
+     */
+    FNV1_64_HASH,
+    /** 
+     * hash based on md5
+     *  **/
+    KETAMA_HASH;
+
+    private static final long FNV_64_INIT  = 0xcbf29ce484222325L;
+    private static final long FNV_64_PRIME = 0x100000001b3L;
+
+    public long hash(final String k) {
+        long rv = 0;
+        switch (this) {
+            case CRC32_HASH:
+                // return (crc32(shift) >> 16) & 0x7fff;
+                CRC32 crc32 = new CRC32();
+                crc32.update(k.getBytes());
+                rv = crc32.getValue() >> 16 & 0x7fff;
+                break;
+            case FNV1_64_HASH: {
+                rv = FNV_64_INIT;
+                int len = k.length();
+                for (int i = 0; i < len; i++) {
+                    rv *= FNV_64_PRIME;
+                    rv ^= k.charAt(i);
+                }
+            }
+                break;
+            case KETAMA_HASH:
+                byte[] bKey = computeMd5(k);
+                rv = (long) (bKey[3] & 0xFF) << 24 | (long) (bKey[2] & 0xFF) << 16 | (long) (bKey[1] & 0xFF) << 8
+                     | bKey[0] & 0xFF;
+                break;
+
+        }
+        return rv;
+    }
+
+    private static ThreadLocal md5Local = new ThreadLocal<>();
+
+    /**
+     * Get the md5 of the given key.
+     */
+    public static byte[] computeMd5(String k) {
+        MessageDigest md5 = md5Local.get();
+        if (md5 == null) {
+            try {
+                md5 = MessageDigest.getInstance("MD5");
+                md5Local.set(md5);
+            } catch (NoSuchAlgorithmException e) {
+                throw new RuntimeException("MD5 not supported", e);
+            }
+        }
+        md5.reset();
+        md5.update(k.getBytes(StandardCharsets.UTF_8));
+        return md5.digest();
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashUtils.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashUtils.java
new file mode 100644
index 0000000..d1b5098
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/HashUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+public class HashUtils {
+
+    public static long getHeadKey(TreeMap map, String key) {
+        long hash = Math.abs(HashAlgorithm.FNV1_64_HASH.hash(key));
+        SortedMap headMap = map.headMap(hash, true);
+        return headMap != null && !headMap.isEmpty() ? headMap.lastKey() : map.firstKey();
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/KeyNotFoundException.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/KeyNotFoundException.java
new file mode 100644
index 0000000..9345fac
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/KeyNotFoundException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic;
+
+/**
+ * Key not found exception
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 2:24:08 PM
+ */
+public class KeyNotFoundException extends Exception {
+
+    private static final long serialVersionUID = 1643417397293554363L;
+
+    public KeyNotFoundException() {
+        super();
+    }
+
+    public KeyNotFoundException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
+        super(message, cause, enableSuppression, writableStackTrace);
+    }
+
+    public KeyNotFoundException(String message, Throwable cause) {
+        super(message, cause);
+    }
+
+    public KeyNotFoundException(String message) {
+        super(message);
+    }
+
+    public KeyNotFoundException(Throwable cause) {
+        super(cause);
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClient.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClient.java
new file mode 100644
index 0000000..d1b3eea
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClient.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.client;
+
+import java.util.List;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeoutException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.RpcClient;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+import com.alipay.sofa.jraft.test.atomic.HashUtils;
+import com.alipay.sofa.jraft.test.atomic.KeyNotFoundException;
+import com.alipay.sofa.jraft.test.atomic.command.BooleanCommand;
+import com.alipay.sofa.jraft.test.atomic.command.CompareAndSetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.GetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.GetSlotsCommand;
+import com.alipay.sofa.jraft.test.atomic.command.IncrementAndGetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.SetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.SlotsResponseCommand;
+import com.alipay.sofa.jraft.test.atomic.command.ValueCommand;
+
+/**
+ * A counter client
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 4:08:14 PM
+ */
+public class AtomicClient {
+
+    static final Logger                LOG    = LoggerFactory.getLogger(AtomicClient.class);
+
+    private final Configuration        conf;
+    private final CliClientServiceImpl cliClientService;
+    private RpcClient                  rpcClient;
+    private CliOptions                 cliOptions;
+    private TreeMap      groups = new TreeMap<>();
+
+    public AtomicClient(String groupId, Configuration conf) {
+        super();
+        this.conf = conf;
+        this.cliClientService = new CliClientServiceImpl();
+    }
+
+    public void shutdown() {
+        this.cliClientService.shutdown();
+        for (final String groupId : groups.values()) {
+            RouteTable.getInstance().removeGroup(groupId);
+        }
+    }
+
+    public void start() throws InterruptedException, TimeoutException {
+        cliOptions = new CliOptions();
+        this.cliClientService.init(cliOptions);
+        this.rpcClient = this.cliClientService.getRpcClient();
+        if (conf != null) {
+            final Set peers = conf.getPeerSet();
+            for (final PeerId peer : peers) {
+                try {
+                    final BooleanCommand cmd = (BooleanCommand) this.rpcClient.invokeSync(peer.getEndpoint(),
+                        new GetSlotsCommand(), cliOptions.getRpcDefaultTimeout());
+                    if (cmd instanceof SlotsResponseCommand) {
+                        groups = ((SlotsResponseCommand) cmd).getMap();
+                        break;
+                    } else {
+                        LOG.warn("Fail to get slots from peer {}, error: {}", peer, cmd.getErrorMsg());
+                    }
+                } catch (final Throwable t) {
+                    LOG.warn("Fail to get slots from peer {}, error: {}", peer, t.getMessage());
+                    //continue;
+                }
+            }
+
+            if (groups == null || groups.isEmpty()) {
+                throw new IllegalArgumentException("Can't get slots from any peers");
+            } else {
+                LOG.info("All groups  is {}", groups);
+            }
+            for (final String groupId : groups.values()) {
+                RouteTable.getInstance().updateConfiguration(groupId, conf);
+                refreshLeader(groupId);
+                refreshConf(groupId);
+            }
+        }
+
+    }
+
+    private void refreshConf(String groupId) throws InterruptedException, TimeoutException {
+        RouteTable.getInstance().refreshConfiguration(cliClientService, groupId, cliOptions.getRpcDefaultTimeout());
+    }
+
+    private void refreshLeader(String groupId) throws InterruptedException, TimeoutException {
+        RouteTable.getInstance().refreshLeader(cliClientService, groupId, cliOptions.getRpcDefaultTimeout());
+    }
+
+    private String getGroupId(String key) {
+        return groups.get(HashUtils.getHeadKey(groups, key));
+    }
+
+    public long get(String key) throws InterruptedException, KeyNotFoundException, TimeoutException {
+        return this.get(getLeaderByKey(key), key, false, false);
+    }
+
+    private PeerId getLeaderByKey(String key) throws InterruptedException, TimeoutException {
+        return getLeader(getGroupId(key));
+    }
+
+    private PeerId getLeader(String key) throws InterruptedException, TimeoutException {
+        final String groupId = getGroupId(key);
+        refreshLeader(groupId);
+        return RouteTable.getInstance().selectLeader(groupId);
+    }
+
+    private PeerId getPeer(String key) throws InterruptedException, TimeoutException {
+        final String groupId = getGroupId(key);
+        this.refreshConf(groupId);
+        final List peers = RouteTable.getInstance().getConfiguration(groupId).getPeers();
+        return peers.get(ThreadLocalRandom.current().nextInt(peers.size()));
+    }
+
+    public long get(String key, boolean readFromQuorum) throws KeyNotFoundException, InterruptedException,
+                                                       TimeoutException {
+        if (readFromQuorum) {
+            return get(getPeer(key), key, true, false);
+        } else {
+            //read from leader
+            return get(key);
+        }
+    }
+
+    public long get(PeerId peer, String key, boolean readFromQuorum, boolean readByStateMachine)
+                                                                                                throws KeyNotFoundException,
+                                                                                                InterruptedException {
+        try {
+            final GetCommand request = new GetCommand(key);
+            request.setReadFromQuorum(readFromQuorum);
+            request.setReadByStateMachine(readByStateMachine);
+            final Object response = this.rpcClient.invokeSync(peer.getEndpoint(), request,
+                cliOptions.getRpcDefaultTimeout());
+            final BooleanCommand cmd = (BooleanCommand) response;
+            if (cmd.isSuccess()) {
+                return ((ValueCommand) cmd).getVlaue();
+            } else {
+                if (cmd.getErrorMsg().equals("key not found")) {
+                    throw new KeyNotFoundException();
+                } else {
+                    throw new IllegalStateException("Server error:" + cmd.getErrorMsg());
+                }
+            }
+        } catch (final Throwable t) {
+            t.printStackTrace();
+            throw new IllegalStateException("Remoting error:" + t.getMessage());
+        }
+    }
+
+    public long addAndGet(String key, long delta) throws InterruptedException, TimeoutException {
+        return this.addAndGet(getLeaderByKey(key), key, delta);
+    }
+
+    public long addAndGet(PeerId peer, String key, long delta) throws InterruptedException {
+        try {
+            final IncrementAndGetCommand request = new IncrementAndGetCommand();
+            request.setKey(key);
+            request.setDetal(delta);
+            final Object response = this.rpcClient.invokeSync(peer.getEndpoint(), request,
+                cliOptions.getRpcDefaultTimeout());
+            final BooleanCommand cmd = (BooleanCommand) response;
+            if (cmd.isSuccess()) {
+                return ((ValueCommand) cmd).getVlaue();
+            } else {
+                throw new IllegalStateException("Server error:" + cmd.getErrorMsg());
+            }
+        } catch (final Throwable t) {
+            throw new IllegalStateException("Remoting error:" + t.getMessage());
+        }
+    }
+
+    public boolean set(String key, long value) throws InterruptedException, TimeoutException {
+        return this.set(getLeaderByKey(key), key, value);
+    }
+
+    public boolean set(PeerId peer, String key, long value) throws InterruptedException {
+        try {
+            final SetCommand request = new SetCommand();
+            request.setKey(key);
+            request.setValue(value);
+            final Object response = this.rpcClient.invokeSync(peer.getEndpoint(), request,
+                cliOptions.getRpcDefaultTimeout());
+            final BooleanCommand cmd = (BooleanCommand) response;
+            return cmd.isSuccess();
+        } catch (final Throwable t) {
+            throw new IllegalStateException("Remoting error:" + t.getMessage());
+        }
+    }
+
+    public boolean compareAndSet(String key, long expect, long newVal) throws InterruptedException, TimeoutException {
+        return this.compareAndSet(getLeaderByKey(key), key, expect, newVal);
+    }
+
+    public boolean compareAndSet(PeerId peer, String key, long expect, long newVal) throws InterruptedException {
+        try {
+            final CompareAndSetCommand request = new CompareAndSetCommand();
+            request.setKey(key);
+            request.setNewValue(newVal);
+            request.setExpect(expect);
+            final Object response = this.rpcClient.invokeSync(peer.getEndpoint(), request,
+                cliOptions.getRpcDefaultTimeout());
+            final BooleanCommand cmd = (BooleanCommand) response;
+            return cmd.isSuccess();
+        } catch (final Throwable t) {
+            throw new IllegalStateException("Remoting error:" + t.getMessage());
+        }
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClientTest.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClientTest.java
new file mode 100644
index 0000000..5787a42
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/client/AtomicClientTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.client;
+
+import java.util.concurrent.CyclicBarrier;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.RouteTable;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.CliOptions;
+import com.alipay.sofa.jraft.rpc.impl.cli.CliClientServiceImpl;
+
+public class AtomicClientTest {
+
+    public static void main(String[] args) throws Exception {
+        final RouteTable table = RouteTable.getInstance();
+        table.updateConfiguration("atomic_0",
+            JRaftUtils.getConfiguration("127.0.0.1:8609,127.0.0.1:8610,127.0.0.1:8611"));
+        final CliClientServiceImpl cliClientService = new CliClientServiceImpl();
+        cliClientService.init(new CliOptions());
+        final Status st = table.refreshLeader(cliClientService, "atomic_0", 10000);
+        System.out.println(st);
+
+        final AtomicClient cli = new AtomicClient("atomic", JRaftUtils.getConfiguration("localhost:8610"));
+        final PeerId leader = table.selectLeader("atomic_0");
+        cli.start();
+
+        final int threads = 30;
+        final int count = 10000;
+        final CyclicBarrier barrier = new CyclicBarrier(threads + 1);
+
+        for (int t = 0; t < threads; t++) {
+            new Thread() {
+                @Override
+                public void run() {
+                    long sum = 0;
+                    try {
+                        barrier.await();
+                        final PeerId peer = new PeerId("localhost", 8611);
+                        for (int i = 0; i < count; i++) {
+                            sum += cli.get(leader, "a", true, false);
+                            //sum += cli.addAndGet(leader, "a", i);
+                        }
+                        barrier.await();
+                    } catch (final Exception e) {
+                        e.printStackTrace();
+                    } finally {
+                        System.out.println("sum=" + sum);
+                    }
+                }
+            }.start();
+
+        }
+        final long start = System.currentTimeMillis();
+        barrier.await();
+        barrier.await();
+        final long cost = System.currentTimeMillis() - start;
+        final long tps = Math.round(threads * count * 1000.0 / cost);
+        System.out.println("tps=" + tps + ",cost=" + cost + " ms.");
+        cli.shutdown();
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BaseRequestCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BaseRequestCommand.java
new file mode 100644
index 0000000..0a07e72
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BaseRequestCommand.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+public class BaseRequestCommand {
+    private String key;
+
+    public String getKey() {
+        return this.key;
+    }
+
+    public void setKey(String key) {
+        this.key = key;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BooleanCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BooleanCommand.java
new file mode 100644
index 0000000..ccd520f
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/BooleanCommand.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * Boolean command represents true or false.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:25:10 PM
+ */
+public class BooleanCommand implements Serializable {
+    private static final long serialVersionUID = 2776110757482798187L;
+    private boolean           success;
+    private String            errorMsg;
+    private String            redirect;
+
+    public String getRedirect() {
+        return this.redirect;
+    }
+
+    public void setRedirect(String redirect) {
+        this.redirect = redirect;
+    }
+
+    public BooleanCommand() {
+        super();
+    }
+
+    public BooleanCommand(boolean result) {
+        this(result, null);
+    }
+
+    public BooleanCommand(boolean result, String errorMsg) {
+        super();
+        this.success = result;
+        this.errorMsg = errorMsg;
+    }
+
+    public String getErrorMsg() {
+        return this.errorMsg;
+    }
+
+    public void setErrorMsg(String errorMsg) {
+        this.errorMsg = errorMsg;
+    }
+
+    public boolean isSuccess() {
+        return this.success;
+    }
+
+    public void setSuccess(boolean result) {
+        this.success = result;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CommandCodec.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CommandCodec.java
new file mode 100644
index 0000000..f92b4f0
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CommandCodec.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import com.alipay.remoting.exception.CodecException;
+import com.alipay.remoting.serialization.SerializerManager;
+
+/**
+ * Command codec
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:30:30 PM
+ */
+public class CommandCodec {
+    /**
+     * encode the command,returns the byte array.
+     * @param obj
+     * @return
+     */
+    public static byte[] encodeCommand(Object obj) {
+        try {
+            return SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(obj);
+        } catch (final CodecException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+
+    /**
+     * Decode the command object from byte array.
+     * @param content
+     * @param clazz
+     * @return
+     */
+    public static  T decodeCommand(byte[] content, Class clazz) {
+        try {
+            return SerializerManager.getSerializer(SerializerManager.Hessian2).deserialize(content, clazz.getName());
+        } catch (final CodecException e) {
+            throw new IllegalStateException(e);
+        }
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CompareAndSetCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CompareAndSetCommand.java
new file mode 100644
index 0000000..d65ed47
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/CompareAndSetCommand.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * compareAndSet(key, expect, newVal)
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:25:48 PM
+ */
+public class CompareAndSetCommand extends BaseRequestCommand implements Serializable {
+    private static final long serialVersionUID = 9094660933580602829L;
+    private long              expect;
+    private long              newValue;
+
+    public long getExpect() {
+        return this.expect;
+    }
+
+    public void setExpect(long expect) {
+        this.expect = expect;
+    }
+
+    public long getNewValue() {
+        return this.newValue;
+    }
+
+    public void setNewValue(long newValue) {
+        this.newValue = newValue;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetCommand.java
new file mode 100644
index 0000000..4f15fbc
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetCommand.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * Get value command
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:28:37 PM
+ */
+public class GetCommand extends BaseRequestCommand implements Serializable {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID   = -512223854448447263L;
+
+    //Whether to use ReadIndex
+    private boolean           readFromQuorum     = false;
+
+    private boolean           readByStateMachine = false;
+
+    public boolean isReadByStateMachine() {
+        return readByStateMachine;
+    }
+
+    public void setReadByStateMachine(boolean readByStateMachine) {
+        this.readByStateMachine = readByStateMachine;
+    }
+
+    public boolean isReadFromQuorum() {
+        return readFromQuorum;
+    }
+
+    public void setReadFromQuorum(boolean readFromQuorum) {
+        this.readFromQuorum = readFromQuorum;
+    }
+
+    public GetCommand() {
+
+    }
+
+    public GetCommand(String key) {
+        setKey(key);
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetSlotsCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetSlotsCommand.java
new file mode 100644
index 0000000..baa64f6
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/GetSlotsCommand.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * Get total slots command
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-May-02 11:36:30 AM
+ */
+public class GetSlotsCommand implements Serializable {
+
+    /**
+     * 
+     */
+    private static final long serialVersionUID = -2970750203095468019L;
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/IncrementAndGetCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/IncrementAndGetCommand.java
new file mode 100644
index 0000000..79b3368
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/IncrementAndGetCommand.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * Increment with detal and get the latest value.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:28:42 PM
+ */
+public class IncrementAndGetCommand extends BaseRequestCommand implements Serializable {
+    private static final long serialVersionUID = -1232443841104358771L;
+    private long              detal;
+
+    public long getDetal() {
+        return this.detal;
+    }
+
+    public void setDetal(long detal) {
+        this.detal = detal;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SetCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SetCommand.java
new file mode 100644
index 0000000..86d9031
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SetCommand.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+
+/**
+ * set value command
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:28:54 PM
+ */
+public class SetCommand extends BaseRequestCommand implements Serializable {
+    private static final long serialVersionUID = 5942385417491201345L;
+
+    private long              value;
+
+    public long getValue() {
+        return this.value;
+    }
+
+    public void setValue(long value) {
+        this.value = value;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SlotsResponseCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SlotsResponseCommand.java
new file mode 100644
index 0000000..14f0010
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/SlotsResponseCommand.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+import java.io.Serializable;
+import java.util.TreeMap;
+
+public class SlotsResponseCommand extends BooleanCommand implements Serializable {
+    private static final long     serialVersionUID = -3155350383161976585L;
+    private TreeMap map              = new TreeMap<>();
+
+    public TreeMap getMap() {
+        return this.map;
+    }
+
+    public void setMap(TreeMap map) {
+        this.map = map;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/ValueCommand.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/ValueCommand.java
new file mode 100644
index 0000000..e678b3f
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/command/ValueCommand.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.command;
+
+/**
+ * Value response command
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:29:03 PM
+ */
+public class ValueCommand extends BooleanCommand {
+    private static final long serialVersionUID = -4313480716428249772L;
+    private long              vlaue;
+
+    public ValueCommand() {
+        super();
+    }
+
+    public ValueCommand(boolean result, String errorMsg) {
+        super(result, errorMsg);
+    }
+
+    public ValueCommand(boolean result) {
+        super(result);
+    }
+
+    public ValueCommand(long vlaue) {
+        super();
+        this.vlaue = vlaue;
+        this.setSuccess(true);
+    }
+
+    public long getVlaue() {
+        return this.vlaue;
+    }
+
+    public void setVlaue(long vlaue) {
+        this.vlaue = vlaue;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicRangeGroup.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicRangeGroup.java
new file mode 100644
index 0000000..6dddd02
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicRangeGroup.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.Node;
+import com.alipay.sofa.jraft.RaftGroupService;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.closure.ReadIndexClosure;
+import com.alipay.sofa.jraft.conf.Configuration;
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.option.NodeOptions;
+import com.alipay.sofa.jraft.option.ReadOnlyOption;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.test.atomic.KeyNotFoundException;
+import com.alipay.sofa.jraft.test.atomic.command.BooleanCommand;
+import com.alipay.sofa.jraft.test.atomic.command.ValueCommand;
+import com.alipay.sofa.jraft.test.atomic.server.processor.GetCommandProcessor;
+import com.alipay.sofa.jraft.util.Bits;
+import com.codahale.metrics.ConsoleReporter;
+
+/**
+ * Atomic range node in a raft group.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 4:51:02 PM
+ */
+public class AtomicRangeGroup {
+
+    final static Logger              LOG       = LoggerFactory.getLogger(AtomicRangeGroup.class);
+
+    private final RaftGroupService   raftGroupService;
+    private final Node               node;
+    private final AtomicStateMachine fsm;
+
+    private final long               minSlot;                                                    //inclusion
+    private final long               maxSlot;                                                    //exclusion
+    private final AtomicInteger      requestId = new AtomicInteger(0);
+
+    public long getMinSlot() {
+        return this.minSlot;
+    }
+
+    public long getMaxSlot() {
+        return this.maxSlot;
+    }
+
+    public AtomicRangeGroup(String dataPath, String groupId, PeerId serverId, long minSlot, long maxSlot,
+                            NodeOptions nodeOptions, RpcServer rpcServer) throws IOException {
+        // Init file path
+        FileUtils.forceMkdir(new File(dataPath));
+        this.minSlot = minSlot;
+        this.maxSlot = maxSlot;
+
+        // Init statemachine
+        this.fsm = new AtomicStateMachine();
+
+        // Set statemachine to bootstrap options
+        nodeOptions.setFsm(this.fsm);
+        nodeOptions.setEnableMetrics(true);
+        nodeOptions.getRaftOptions().setReplicatorPipeline(true);
+        nodeOptions.getRaftOptions().setSync(true);
+        nodeOptions.getRaftOptions().setReadOnlyOptions(ReadOnlyOption.ReadOnlySafe);
+
+        // Set the data path
+        // Log, required
+        nodeOptions.setLogUri(dataPath + File.separator + "log");
+        // Metadata, required
+        nodeOptions.setRaftMetaUri(dataPath + File.separator + "raft_meta");
+        // Snapshot, not required, but recommend
+        nodeOptions.setSnapshotUri(dataPath + File.separator + "snapshot");
+        // Init raft group service framework
+        this.raftGroupService = new RaftGroupService(groupId, serverId, nodeOptions, rpcServer);
+        // Startup node
+        this.node = this.raftGroupService.start();
+
+        final ConsoleReporter reporter = ConsoleReporter.forRegistry(node.getNodeMetrics().getMetricRegistry())
+            .convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MILLISECONDS).build();
+        reporter.start(60, TimeUnit.SECONDS);
+
+    }
+
+    public void readFromQuorum(final String key, RpcContext asyncContext) {
+        final byte[] reqContext = new byte[4];
+        Bits.putInt(reqContext, 0, requestId.incrementAndGet());
+        this.node.readIndex(reqContext, new ReadIndexClosure() {
+
+            @Override
+            public void run(Status status, long index, byte[] reqCtx) {
+                if (status.isOk()) {
+                    try {
+                        asyncContext.sendResponse(new ValueCommand(fsm.getValue(key)));
+                    } catch (final KeyNotFoundException e) {
+                        asyncContext.sendResponse(GetCommandProcessor.createKeyNotFoundResponse());
+                    }
+                } else {
+                    asyncContext.sendResponse(new BooleanCommand(false, status.getErrorMsg()));
+                }
+            }
+        });
+    }
+
+    public AtomicStateMachine getFsm() {
+        return this.fsm;
+    }
+
+    public Node getNode() {
+        return this.node;
+    }
+
+    public RaftGroupService RaftGroupService() {
+        return this.raftGroupService;
+    }
+
+    /**
+     * Redirect request to new leader
+     * @return
+     */
+    public BooleanCommand redirect() {
+        final BooleanCommand response = new BooleanCommand();
+        response.setSuccess(false);
+        response.setErrorMsg("Not leader");
+        if (node != null) {
+            final PeerId leader = node.getLeaderId();
+            if (leader != null) {
+                response.setRedirect(leader.toString());
+            }
+        }
+
+        return response;
+    }
+
+    public static AtomicRangeGroup start(StartupConf conf, RpcServer rpcServer) throws IOException {
+
+        final NodeOptions nodeOptions = new NodeOptions();
+        // Set election timeout to 1 second
+        nodeOptions.setElectionTimeoutMs(1000);
+        // Close cli service
+        nodeOptions.setDisableCli(false);
+        // A snapshot saving would be triggered every 30 seconds
+        // nodeOptions.setSnapshotIntervalSecs(30);
+        // Parsing Options
+        final PeerId serverId = new PeerId();
+        if (!serverId.parse(conf.getServerAddress())) {
+            throw new IllegalArgumentException("Fail to parse serverId:" + conf.getServerAddress());
+        }
+        final Configuration initConf = new Configuration();
+        if (!initConf.parse(conf.getConf())) {
+            throw new IllegalArgumentException("Fail to parse initConf:" + conf.getConf());
+        }
+        // Set the initial cluster configuration
+        nodeOptions.setInitialConf(initConf);
+        // Startup node
+        final AtomicRangeGroup node = new AtomicRangeGroup(conf.getDataPath(), conf.getGroupId(), serverId,
+            conf.getMinSlot(), conf.getMaxSlot(), nodeOptions, rpcServer);
+        LOG.info("Started range node[{}-{}] at port:{}", conf.getMinSlot(), conf.getMaxSlot(), node.getNode()
+            .getNodeId().getPeerId().getPort());
+        return node;
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicServer.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicServer.java
new file mode 100644
index 0000000..f58e90f
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicServer.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.TreeMap;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.entity.PeerId;
+import com.alipay.sofa.jraft.rpc.RaftRpcServerFactory;
+import com.alipay.sofa.jraft.rpc.RpcServer;
+import com.alipay.sofa.jraft.test.atomic.HashUtils;
+import com.alipay.sofa.jraft.test.atomic.server.processor.CompareAndSetCommandProcessor;
+import com.alipay.sofa.jraft.test.atomic.server.processor.GetCommandProcessor;
+import com.alipay.sofa.jraft.test.atomic.server.processor.GetSlotsCommandProcessor;
+import com.alipay.sofa.jraft.test.atomic.server.processor.IncrementAndGetCommandProcessor;
+import com.alipay.sofa.jraft.test.atomic.server.processor.SetCommandProcessor;
+
+/**
+ * Atomic server with multi raft groups.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-May-02 10:50:14 AM
+ */
+public class AtomicServer {
+
+    private static final Logger             LOG    = LoggerFactory.getLogger(AtomicServer.class);
+
+    private TreeMap nodes  = new TreeMap<>();
+    private TreeMap           groups = new TreeMap<>();
+    private int                             totalSlots;
+    private StartupConf                     conf;
+
+    public AtomicRangeGroup getGroupBykey(String key) {
+        return nodes.get(HashUtils.getHeadKey(this.nodes, key));
+    }
+
+    public int getTotalSlots() {
+        return this.totalSlots;
+    }
+
+    public AtomicServer(StartupConf conf) throws IOException {
+        this.conf = conf;
+        LOG.info("Starting atomic server with conf: {}", this.conf);
+        this.totalSlots = conf.getTotalSlots();
+    }
+
+    public TreeMap getGroups() {
+        return this.groups;
+    }
+
+    public void start() throws IOException {
+        PeerId serverId = new PeerId();
+        if (!serverId.parse(conf.getServerAddress())) {
+            throw new IllegalArgumentException("Fail to parse serverId:" + conf.getServerAddress());
+        }
+
+        FileUtils.forceMkdir(new File(conf.getDataPath()));
+        // The same in-process raft group shares the same RPC Server.
+        RpcServer rpcServer = RaftRpcServerFactory.createRaftRpcServer(serverId.getEndpoint());
+        // Register biz handler
+        rpcServer.registerProcessor(new GetSlotsCommandProcessor(this));
+        rpcServer.registerProcessor(new GetCommandProcessor(this));
+        rpcServer.registerProcessor(new IncrementAndGetCommandProcessor(this));
+        rpcServer.registerProcessor(new CompareAndSetCommandProcessor(this));
+        rpcServer.registerProcessor(new SetCommandProcessor(this));
+
+        long step = conf.getMaxSlot() / totalSlots;
+        if (conf.getMaxSlot() % totalSlots > 0) {
+            step = step + 1;
+        }
+        for (int i = 0; i < totalSlots; i++) {
+            long min = i * step;
+            long mayMax = (i + 1) * step;
+            long max = mayMax > conf.getMaxSlot() || mayMax <= 0 ? conf.getMaxSlot() : mayMax;
+            StartupConf nodeConf = new StartupConf();
+            String nodeDataPath = conf.getDataPath() + File.separator + i;
+            nodeConf.setDataPath(nodeDataPath);
+            String nodeGroup = conf.getGroupId() + "_" + i;
+            nodeConf.setGroupId(nodeGroup);
+            nodeConf.setMaxSlot(max);
+            nodeConf.setMinSlot(min);
+            nodeConf.setConf(conf.getConf());
+            nodeConf.setServerAddress(conf.getServerAddress());
+            nodeConf.setTotalSlots(conf.getTotalSlots());
+            LOG.info("Starting range node {}-{} with conf {}", min, max, nodeConf);
+            nodes.put(i * step, AtomicRangeGroup.start(nodeConf, rpcServer));
+            groups.put(i * step, nodeGroup);
+        }
+    }
+
+    public static void start(String confFilePath) throws IOException {
+        StartupConf conf = new StartupConf();
+        if (!conf.loadFromFile(confFilePath)) {
+            throw new IllegalStateException("Load startup config from " + confFilePath + " failed");
+        }
+        AtomicServer server = new AtomicServer(conf);
+        server.start();
+    }
+
+    //for test
+    public static void main(String[] arsg) throws Exception {
+        start("config/server.properties");
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicSnapshotFile.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicSnapshotFile.java
new file mode 100644
index 0000000..9876648
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicSnapshotFile.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.alipay.sofa.jraft.test.atomic.command.CommandCodec;
+
+/**
+ * atomic snapshot file
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-09 5:14:55 PM
+ */
+public class AtomicSnapshotFile {
+    private static final Logger LOG = LoggerFactory.getLogger(AtomicSnapshotFile.class);
+    private String              path;
+
+    public AtomicSnapshotFile(String path) {
+        super();
+        this.path = path;
+    }
+
+    public String getPath() {
+        return this.path;
+    }
+
+    /**
+     * Save value to snapshot file.
+     * @param values
+     * @return
+     */
+    public boolean save(Map values) {
+        try {
+            FileUtils.writeByteArrayToFile(new File(path), CommandCodec.encodeCommand(values));
+            return true;
+        } catch (IOException e) {
+            LOG.error("Fail to save snapshot", e);
+            return false;
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public Map load() throws IOException {
+        byte[] bs = FileUtils.readFileToByteArray(new File(path));
+        if (bs != null && bs.length > 0) {
+            return CommandCodec.decodeCommand(bs, Map.class);
+        }
+        throw new IOException("Fail to load snapshot from " + path + ",content: " + Arrays.toString(bs));
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicStateMachine.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicStateMachine.java
new file mode 100644
index 0000000..f74a80d
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/AtomicStateMachine.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Iterator;
+import com.alipay.sofa.jraft.Status;
+import com.alipay.sofa.jraft.core.StateMachineAdapter;
+import com.alipay.sofa.jraft.error.RaftError;
+import com.alipay.sofa.jraft.error.RaftException;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotReader;
+import com.alipay.sofa.jraft.storage.snapshot.SnapshotWriter;
+import com.alipay.sofa.jraft.test.atomic.KeyNotFoundException;
+import com.alipay.sofa.jraft.test.atomic.command.BaseRequestCommand;
+import com.alipay.sofa.jraft.test.atomic.command.BooleanCommand;
+import com.alipay.sofa.jraft.test.atomic.command.CommandCodec;
+import com.alipay.sofa.jraft.test.atomic.command.CompareAndSetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.GetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.IncrementAndGetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.SetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.ValueCommand;
+import com.alipay.sofa.jraft.util.Utils;
+
+/**
+ * Atomic state machine
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:47:50 PM
+ */
+public class AtomicStateMachine extends StateMachineAdapter {
+
+    private static final Logger                         LOG        = LoggerFactory.getLogger(AtomicStateMachine.class);
+
+    // 
+    private final ConcurrentHashMap counters   = new ConcurrentHashMap<>();
+
+    /**
+     * leader term
+     */
+    private final AtomicLong                            leaderTerm = new AtomicLong(-1);
+
+    public boolean isLeader() {
+        return this.leaderTerm.get() > 0;
+    }
+
+    @Override
+    public void onApply(final Iterator iter) {
+        while (iter.hasNext()) {
+            final Closure done = iter.done();
+            CommandType cmdType;
+            final ByteBuffer data = iter.getData();
+            Object cmd = null;
+            LeaderTaskClosure closure = null;
+            if (done != null) {
+                closure = (LeaderTaskClosure) done;
+                cmdType = closure.getCmdType();
+                cmd = closure.getCmd();
+            } else {
+                final byte b = data.get();
+                final byte[] cmdBytes = new byte[data.remaining()];
+                data.get(cmdBytes);
+                cmdType = CommandType.parseByte(b);
+                switch (cmdType) {
+                    case GET:
+                        cmd = CommandCodec.decodeCommand(cmdBytes, GetCommand.class);
+                        break;
+                    case SET:
+                        cmd = CommandCodec.decodeCommand(cmdBytes, SetCommand.class);
+                        break;
+                    case CAS:
+                        cmd = CommandCodec.decodeCommand(cmdBytes, CompareAndSetCommand.class);
+                        break;
+                    case INC:
+                        cmd = CommandCodec.decodeCommand(cmdBytes, IncrementAndGetCommand.class);
+                        break;
+                }
+            }
+            final String key = ((BaseRequestCommand) cmd).getKey();
+            final AtomicLong counter = getCounter(key, true);
+            Object response = null;
+            switch (cmdType) {
+                case GET:
+                    response = new ValueCommand(counter.get());
+                    break;
+                case SET:
+                    final SetCommand setCmd = (SetCommand) cmd;
+                    counter.set(setCmd.getValue());
+                    response = new BooleanCommand(true);
+                    break;
+                case CAS:
+                    final CompareAndSetCommand casCmd = (CompareAndSetCommand) cmd;
+                    response = new BooleanCommand(counter.compareAndSet(casCmd.getExpect(), casCmd.getNewValue()));
+                    break;
+                case INC:
+                    final IncrementAndGetCommand incCmd = (IncrementAndGetCommand) cmd;
+                    final long ret = counter.addAndGet(incCmd.getDetal());
+                    response = new ValueCommand(ret);
+                    break;
+            }
+            if (closure != null) {
+                closure.setResponse(response);
+                closure.run(Status.OK());
+            }
+            iter.next();
+        }
+    }
+
+    private AtomicLong getCounter(final String key, final boolean createWhenNotFound) {
+        AtomicLong ret = this.counters.get(key);
+        if (ret == null && createWhenNotFound) {
+            ret = new AtomicLong(0);
+            final AtomicLong old = this.counters.putIfAbsent(key, ret);
+            if (old != null) {
+                ret = old;
+            }
+        }
+        return ret;
+    }
+
+    public long getValue(final String key) throws KeyNotFoundException {
+        final AtomicLong counter = getCounter(key, false);
+        if (counter == null) {
+            throw new KeyNotFoundException("Key `" + key + "` not found");
+        }
+        return counter.get();
+    }
+
+    @Override
+  public void onSnapshotSave(final SnapshotWriter writer, final Closure done) {
+    final Map values = new HashMap<>();
+    for (final Map.Entry entry : this.counters.entrySet()) {
+      values.put(entry.getKey(), entry.getValue().get());
+    }
+    Utils.runInThread(() -> {
+      final AtomicSnapshotFile snapshot = new AtomicSnapshotFile(writer.getPath() + File.separator + "data");
+      if (snapshot.save(values)) {
+        if (writer.addFile("data")) {
+          done.run(Status.OK());
+        } else {
+          done.run(new Status(RaftError.EIO, "Fail to add file to writer"));
+        }
+      } else {
+        done.run(new Status(RaftError.EIO, "Fail to save counter snapshot %s", snapshot.getPath()));
+      }
+    });
+  }
+
+    @Override
+    public void onError(final RaftException e) {
+        LOG.error("Raft error: {}", e, e);
+    }
+
+    @Override
+    public boolean onSnapshotLoad(final SnapshotReader reader) {
+        if (isLeader()) {
+            LOG.warn("Leader is not supposed to load snapshot");
+            return false;
+        }
+        if (reader.getFileMeta("data") == null) {
+            LOG.error("Fail to find data file in {}", reader.getPath());
+            return false;
+        }
+        final AtomicSnapshotFile snapshot = new AtomicSnapshotFile(reader.getPath() + File.separator + "data");
+        try {
+            final Map values = snapshot.load();
+            this.counters.clear();
+            if (values != null) {
+                for (final Map.Entry entry : values.entrySet()) {
+                    this.counters.put(entry.getKey(), new AtomicLong(entry.getValue()));
+                }
+            }
+            return true;
+        } catch (final IOException e) {
+            LOG.error("Fail to load snapshot from {}", snapshot.getPath());
+            return false;
+        }
+
+    }
+
+    @Override
+    public void onLeaderStart(final long term) {
+        this.leaderTerm.set(term);
+        super.onLeaderStart(term);
+
+    }
+
+    @Override
+    public void onLeaderStop(final Status status) {
+        this.leaderTerm.set(-1);
+        super.onLeaderStop(status);
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/CommandType.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/CommandType.java
new file mode 100644
index 0000000..a8dead7
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/CommandType.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+/**
+ * command type
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:34:15 PM
+ */
+public enum CommandType {
+    SET, CAS, INC, GET;
+
+    public byte toByte() {
+        switch (this) {
+            case SET:
+                return (byte) 0;
+            case CAS:
+                return (byte) 1;
+            case INC:
+                return (byte) 2;
+            case GET:
+                return (byte) 3;
+        }
+        throw new IllegalArgumentException();
+    }
+
+    public static CommandType parseByte(byte b) {
+        switch (b) {
+            case 0:
+                return SET;
+            case 1:
+                return CAS;
+            case 2:
+                return INC;
+            case 3:
+                return GET;
+        }
+        throw new IllegalArgumentException();
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/LeaderTaskClosure.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/LeaderTaskClosure.java
new file mode 100644
index 0000000..f5dffe0
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/LeaderTaskClosure.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import com.alipay.sofa.jraft.Closure;
+import com.alipay.sofa.jraft.Status;
+
+/**
+ * Leader closure to apply task.
+ * @author boyan (boyan@alibaba-inc.com)
+ *
+ * 2018-Apr-25 1:35:11 PM
+ */
+public class LeaderTaskClosure implements Closure {
+    private Object      cmd;
+    private CommandType cmdType;
+    private Closure     done;
+    private Object      response;
+
+    @Override
+    public void run(Status status) {
+        if (this.done != null) {
+            done.run(status);
+        }
+    }
+
+    public Object getResponse() {
+        return this.response;
+    }
+
+    public void setResponse(Object response) {
+        this.response = response;
+    }
+
+    public Object getCmd() {
+        return this.cmd;
+    }
+
+    public void setCmd(Object cmd) {
+        this.cmd = cmd;
+    }
+
+    public CommandType getCmdType() {
+        return this.cmdType;
+    }
+
+    public void setCmdType(CommandType cmdType) {
+        this.cmdType = cmdType;
+    }
+
+    public Closure getDone() {
+        return this.done;
+    }
+
+    public void setDone(Closure done) {
+        this.done = done;
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/StartupConf.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/StartupConf.java
new file mode 100644
index 0000000..c8b9878
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/StartupConf.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.alipay.sofa.jraft.JRaftUtils;
+import com.alipay.sofa.jraft.conf.Configuration;
+
+public class StartupConf {
+    private String groupId;
+    private String dataPath;
+    private String conf;
+    private String serverAddress;
+    private long   minSlot;
+    private long   maxSlot;
+    private int    totalSlots = 1;
+
+    public int getTotalSlots() {
+        return this.totalSlots;
+    }
+
+    public void setTotalSlots(int totalSlots) {
+        this.totalSlots = totalSlots;
+    }
+
+    public long getMinSlot() {
+        return this.minSlot;
+    }
+
+    public void setMinSlot(long minSlot) {
+        this.minSlot = minSlot;
+    }
+
+    public long getMaxSlot() {
+        return this.maxSlot;
+    }
+
+    public void setMaxSlot(long maxSlot) {
+        this.maxSlot = maxSlot;
+    }
+
+    public boolean loadFromFile(String file) throws IOException {
+        try (FileInputStream fin = new FileInputStream(new File(file))) {
+            Properties props = new Properties();
+            props.load(fin);
+            this.groupId = props.getProperty("groupId");
+            this.dataPath = props.getProperty("dataPath", "/tmp/atomic");
+            this.conf = props.getProperty("conf");
+            this.serverAddress = props.getProperty("serverAddress");
+            this.minSlot = Long.valueOf(props.getProperty("minSlot", "0"));
+            this.maxSlot = Long.valueOf(props.getProperty("maxSlot", String.valueOf(Long.MAX_VALUE)));
+            this.totalSlots = Integer.valueOf(props.getProperty("totalSlots", "1"));
+            return this.verify();
+        }
+    }
+
+    private boolean verify() {
+        if (StringUtils.isBlank(groupId)) {
+            throw new IllegalArgumentException("Blank groupId");
+        }
+        if (StringUtils.isBlank(dataPath)) {
+            throw new IllegalArgumentException("Blank dataPath");
+        }
+        if (StringUtils.isBlank(conf)) {
+            throw new IllegalArgumentException("Blank conf");
+        }
+        Configuration initConf = JRaftUtils.getConfiguration(conf);
+        if (initConf.isEmpty()) {
+            throw new IllegalArgumentException("Blank conf");
+        }
+        if (minSlot < 0) {
+            throw new IllegalArgumentException("Invalid min slot");
+        }
+        if (minSlot > maxSlot) {
+            throw new IllegalArgumentException("Invalid slot range.");
+        }
+        if (this.totalSlots <= 0) {
+            throw new IllegalArgumentException("Invalid total slots");
+        }
+        return true;
+    }
+
+    public String getGroupId() {
+        return this.groupId;
+    }
+
+    public void setGroupId(String groupId) {
+        this.groupId = groupId;
+    }
+
+    public String getDataPath() {
+        return this.dataPath;
+    }
+
+    public void setDataPath(String dataPath) {
+        this.dataPath = dataPath;
+    }
+
+    public String getConf() {
+        return this.conf;
+    }
+
+    public void setConf(String conf) {
+        this.conf = conf;
+    }
+
+    public String getServerAddress() {
+        return this.serverAddress;
+    }
+
+    public void setServerAddress(String serverAddress) {
+        this.serverAddress = serverAddress;
+    }
+
+    @Override
+    public String toString() {
+        return "StartupConf [groupId=" + this.groupId + ", dataPath=" + this.dataPath + ", conf=" + this.conf
+               + ", serverAddress=" + this.serverAddress + ", minSlot=" + this.minSlot + ", maxSlot=" + this.maxSlot
+               + ", totalSlots=" + this.totalSlots + "]";
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/BaseAsyncUserProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/BaseAsyncUserProcessor.java
new file mode 100644
index 0000000..8474fd7
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/BaseAsyncUserProcessor.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import java.nio.ByteBuffer;
+
+import com.alipay.sofa.jraft.entity.Task;
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.test.atomic.command.BaseRequestCommand;
+import com.alipay.sofa.jraft.test.atomic.command.BooleanCommand;
+import com.alipay.sofa.jraft.test.atomic.command.CommandCodec;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicRangeGroup;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+import com.alipay.sofa.jraft.test.atomic.server.CommandType;
+import com.alipay.sofa.jraft.test.atomic.server.LeaderTaskClosure;
+
+public abstract class BaseAsyncUserProcessor implements RpcProcessor {
+    protected AtomicServer server;
+
+    public BaseAsyncUserProcessor(AtomicServer server) {
+        super();
+        this.server = server;
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final T request) {
+        final AtomicRangeGroup group = server.getGroupBykey(request.getKey());
+        if (!group.getFsm().isLeader()) {
+            rpcCtx.sendResponse(group.redirect());
+            return;
+        }
+
+        final CommandType cmdType = getCmdType();
+        final Task task = createTask(rpcCtx, request, cmdType);
+        group.getNode().apply(task);
+    }
+
+    protected abstract CommandType getCmdType();
+
+    private Task createTask(RpcContext asyncCtx, T request, CommandType cmdType) {
+        final LeaderTaskClosure closure = new LeaderTaskClosure();
+        closure.setCmd(request);
+        closure.setCmdType(cmdType);
+        closure.setDone(status -> {
+            if (status.isOk()) {
+                asyncCtx.sendResponse(closure.getResponse());
+            } else {
+                asyncCtx.sendResponse(new BooleanCommand(false, status.getErrorMsg()));
+            }
+        });
+        final byte[] cmdBytes = CommandCodec.encodeCommand(request);
+        final ByteBuffer data = ByteBuffer.allocate(cmdBytes.length + 1);
+        data.put(cmdType.toByte());
+        data.put(cmdBytes);
+        data.flip();
+        return new Task(data, closure);
+    }
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/CompareAndSetCommandProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/CompareAndSetCommandProcessor.java
new file mode 100644
index 0000000..6c933e9
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/CompareAndSetCommandProcessor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import com.alipay.sofa.jraft.test.atomic.command.CompareAndSetCommand;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+import com.alipay.sofa.jraft.test.atomic.server.CommandType;
+
+public class CompareAndSetCommandProcessor extends BaseAsyncUserProcessor {
+
+    @Override
+    protected CommandType getCmdType() {
+        return CommandType.CAS;
+    }
+
+    public CompareAndSetCommandProcessor(AtomicServer server) {
+        super(server);
+    }
+
+    @Override
+    public String interest() {
+        return CompareAndSetCommand.class.getName();
+    }
+
+}
\ No newline at end of file
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetCommandProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetCommandProcessor.java
new file mode 100644
index 0000000..cdef71e
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetCommandProcessor.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.test.atomic.KeyNotFoundException;
+import com.alipay.sofa.jraft.test.atomic.command.GetCommand;
+import com.alipay.sofa.jraft.test.atomic.command.ValueCommand;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicRangeGroup;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+import com.alipay.sofa.jraft.test.atomic.server.CommandType;
+
+/**
+ * Get command processor
+ * @author dennis
+ *
+ */
+public class GetCommandProcessor extends BaseAsyncUserProcessor {
+
+    @Override
+    protected CommandType getCmdType() {
+        return CommandType.GET;
+    }
+
+    public GetCommandProcessor(AtomicServer server) {
+        super(server);
+    }
+
+    @Override
+    public void handleRequest(RpcContext rpcCtx, GetCommand request) {
+        if (request.isReadByStateMachine()) {
+            super.handleRequest(rpcCtx, request);
+        } else {
+            try {
+                final AtomicRangeGroup group = server.getGroupBykey(request.getKey());
+                if (!request.isReadFromQuorum()) {
+                    rpcCtx.sendResponse(new ValueCommand(group.getFsm().getValue(request.getKey())));
+                } else {
+                    group.readFromQuorum(request.getKey(), rpcCtx);
+                }
+            } catch (final KeyNotFoundException e) {
+                rpcCtx.sendResponse(createKeyNotFoundResponse());
+            }
+        }
+    }
+
+    public static ValueCommand createKeyNotFoundResponse() {
+        return new ValueCommand(false, "key not found");
+    }
+
+    @Override
+    public String interest() {
+        return GetCommand.class.getName();
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetSlotsCommandProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetSlotsCommandProcessor.java
new file mode 100644
index 0000000..a2162bf
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/GetSlotsCommandProcessor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import com.alipay.sofa.jraft.rpc.RpcContext;
+import com.alipay.sofa.jraft.rpc.RpcProcessor;
+import com.alipay.sofa.jraft.test.atomic.command.GetSlotsCommand;
+import com.alipay.sofa.jraft.test.atomic.command.SlotsResponseCommand;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+
+public class GetSlotsCommandProcessor implements RpcProcessor {
+    private AtomicServer server;
+
+    public GetSlotsCommandProcessor(AtomicServer server) {
+        super();
+        this.server = server;
+    }
+
+    @Override
+    public void handleRequest(final RpcContext rpcCtx, final GetSlotsCommand request) {
+        final SlotsResponseCommand response = new SlotsResponseCommand();
+        response.setMap(this.server.getGroups());
+        rpcCtx.sendResponse(response);
+    }
+
+    @Override
+    public String interest() {
+        return GetSlotsCommand.class.getName();
+    }
+
+}
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/IncrementAndGetCommandProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/IncrementAndGetCommandProcessor.java
new file mode 100644
index 0000000..4921b17
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/IncrementAndGetCommandProcessor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import com.alipay.sofa.jraft.test.atomic.command.IncrementAndGetCommand;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+import com.alipay.sofa.jraft.test.atomic.server.CommandType;
+
+public class IncrementAndGetCommandProcessor extends BaseAsyncUserProcessor {
+
+    @Override
+    protected CommandType getCmdType() {
+        return CommandType.INC;
+    }
+
+    public IncrementAndGetCommandProcessor(AtomicServer server) {
+        super(server);
+    }
+
+    @Override
+    public String interest() {
+        return IncrementAndGetCommand.class.getName();
+    }
+
+}
\ No newline at end of file
diff --git a/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/SetCommandProcessor.java b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/SetCommandProcessor.java
new file mode 100644
index 0000000..59b3638
--- /dev/null
+++ b/jraft-test/src/main/java/com/alipay/sofa/jraft/test/atomic/server/processor/SetCommandProcessor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.alipay.sofa.jraft.test.atomic.server.processor;
+
+import com.alipay.sofa.jraft.test.atomic.command.SetCommand;
+import com.alipay.sofa.jraft.test.atomic.server.AtomicServer;
+import com.alipay.sofa.jraft.test.atomic.server.CommandType;
+
+public class SetCommandProcessor extends BaseAsyncUserProcessor {
+
+    @Override
+    protected CommandType getCmdType() {
+        return CommandType.SET;
+    }
+
+    public SetCommandProcessor(AtomicServer server) {
+        super(server);
+    }
+
+    @Override
+    public String interest() {
+        return SetCommand.class.getName();
+    }
+
+}
\ No newline at end of file
diff --git a/jraft-test/src/main/resources/log4j2.xml b/jraft-test/src/main/resources/log4j2.xml
new file mode 100644
index 0000000..ffcac83
--- /dev/null
+++ b/jraft-test/src/main/resources/log4j2.xml
@@ -0,0 +1,25 @@
+
+
+ 
+	
+		
+			
+		
+ 
+		
+			
+			
+				
+			
+			
+		
+ 
+	
+	
+		
+			
+			
+		
+	
+
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..a7d3699
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,532 @@
+
+
+    4.0.0
+
+    com.alipay.sofa
+    jraft-parent
+    1.3.10.bugfix
+    pom
+
+    ${project.groupId}:${project.artifactId}
+    A production-grade java implementation of RAFT consensus algorithm.
+    https://github.com/alipay/sofa-jraft
+
+    
+        
+            The Apache License, Version 2.0
+            http://www.apache.org/licenses/LICENSE-2.0.txt
+        
+    
+
+    
+        
+            boyan
+            boyan@antfin.com
+            Ant Financial
+            https://www.alipay.com/
+        
+        
+            jiachun
+            jiachun.fjc@alibaba-inc.com
+            Ant Financial
+            https://www.alipay.com/
+        
+    
+
+    
+        jraft-core
+        jraft-test
+        jraft-example
+        jraft-rheakv
+        jraft-extension
+    
+
+    
+        scm:git:git://github.com/alipay/sofa-jraft.git
+        scm:git:ssh://github.com/alipay/sofa-jraft.git
+        http://github.com/alipay/sofa-jraft/tree/master
+    
+
+    
+        3.1.7
+        6.0
+        1.6.4
+        1.21
+        2.8.0
+        2.6
+        3.3.7
+        1.3
+        3.3.6
+        ${project.build.directory}/jacoco.exec
+        true
+        1.8
+        1.8
+        2.1.1
+        1.20
+        5.5.0
+        3.0.2
+        4.8.2
+        4.13.1
+        2.17.1
+        ${user.dir}
+        4.0.2
+        1.9.5
+        1.6.0
+        UTF-8
+        UTF-8
+        3.5.1
+        1.6.0
+        6.22.1.1
+        1.7.21
+    
+
+    
+
+        
+            
+            
+                ${project.groupId}
+                jraft-core
+                ${project.version}
+            
+            
+                ${project.groupId}
+                jraft-test
+                ${project.version}
+            
+            
+                ${project.groupId}
+                jraft-example
+                ${project.version}
+            
+            
+                ${project.groupId}
+                rpc-grpc-impl
+                ${project.version}
+            
+            
+                ${project.groupId}
+                jraft-rheakv-core
+                ${project.version}
+            
+            
+                ${project.groupId}
+                jraft-rheakv-pd
+                ${project.version}
+            
+            
+            
+                org.ow2.asm
+                asm
+                ${asm.version}
+            
+            
+            
+                com.google.code.findbugs
+                jsr305
+                ${jsr305.version}
+            
+            
+            
+                com.alipay.sofa
+                bolt
+                ${bolt.version}
+            
+            
+                com.alipay.sofa
+                hessian
+                ${hessian.version}
+            
+            
+            
+                org.slf4j
+                slf4j-api
+                ${slf4j.version}
+            
+            
+            
+                org.apache.logging.log4j
+                log4j-api
+                ${log4j.version}
+            
+            
+                org.apache.logging.log4j
+                log4j-core
+                ${log4j.version}
+            
+            
+                org.apache.logging.log4j
+                log4j-slf4j-impl
+                ${log4j.version}
+            
+            
+                org.apache.logging.log4j
+                log4j-jcl
+                ${log4j.version}
+            
+            
+            
+                com.lmax
+                disruptor
+                ${disruptor.version}
+            
+            
+                commons-io
+                commons-io
+                ${commons.io.version}
+            
+            
+                org.apache.commons
+                commons-compress
+                ${commons.compress.version}
+            
+            
+                commons-lang
+                commons-lang
+                ${commons.lang.version}
+            
+            
+            
+                com.google.protobuf
+                protobuf-java
+                ${protobuf.version}
+            
+            
+            
+                io.protostuff
+                protostuff-core
+                ${protostuff.version}
+            
+            
+                io.protostuff
+                protostuff-runtime
+                ${protostuff.version}
+            
+            
+            
+                org.rocksdb
+                rocksdbjni
+                ${rocksdb.version}
+            
+            
+            
+                net.openhft
+                affinity
+                ${affinity.version}
+                
+                    
+                        slf4j-api
+                        org.slf4j
+                    
+                
+            
+            
+            
+                net.java.dev.jna
+                jna
+                ${jna.version}
+            
+            
+            
+                org.jctools
+                jctools-core
+                ${jctools.version}
+            
+            
+            
+                io.dropwizard.metrics
+                metrics-core
+                ${metrics.version}
+                
+                    
+                        slf4j-api
+                        org.slf4j
+                    
+                
+            
+            
+            
+            
+            
+            
+            
+            
+                org.openjdk.jmh
+                jmh-core
+                ${jmh.version}
+                test
+            
+            
+                org.openjdk.jmh
+                jmh-generator-annprocess
+                ${jmh.version}
+                test
+            
+            
+            
+                junit
+                junit
+                ${junit.version}
+                test
+            
+            
+                junit
+                junit-dep
+                ${junit.dep.version}
+                test
+            
+            
+                org.hamcrest
+                hamcrest-library
+                ${hamcrest.version}
+                test
+            
+            
+                org.mockito
+                mockito-all
+                ${mockito.version}
+                test
+            
+            
+                org.powermock
+                powermock-api-mockito
+                ${powermock.version}
+                test
+            
+            
+                org.powermock
+                powermock-module-junit4
+                ${powermock.version}
+                test
+            
+        
+
+    
+
+    
+        
+            
+            
+            
+                org.jacoco
+                jacoco-maven-plugin
+                0.7.9
+                
+                    
+                        pre-test
+                        process-classes
+                        
+                            prepare-agent
+                        
+                        
+                            coverageAgent
+                        
+                    
+                
+                
+                    ${jacoco.skip}
+                    ${jacoco.path}
+                    jacoco_coverage
+                
+            
+            
+                org.apache.maven.plugins
+                maven-compiler-plugin
+                3.8.0
+                
+                    ${java.source.version}
+                    ${java.target.version}
+                    ${project.encoding}
+                    true
+                    true
+                
+            
+            
+                org.apache.maven.plugins
+                maven-assembly-plugin
+                
+                    jraft
+                    
+                        assembly.xml
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-surefire-plugin
+                2.12.4
+                true
+                
+                    
+                    -Xmx2048m ${coverageAgent}
+                    
+                        **/*Test.java
+                        **/*TestSuite*.java
+                    
+                    pertest
+                
+            
+            
+                org.apache.maven.plugins
+                maven-source-plugin
+                2.0.2
+                
+                    
+                        attach-sources
+                        
+                            jar
+                        
+                    
+                
+            
+            
+                org.apache.maven.plugins
+                maven-eclipse-plugin
+                2.5.1
+                
+                    true
+                
+            
+            
+            
+                com.googlecode.maven-java-formatter-plugin
+                maven-java-formatter-plugin
+                0.4
+                
+                    
+                        
+                            format
+                        
+                    
+                
+                
+                    ${main.user.dir}/tools/codestyle/formatter.xml
+                    ${project.encoding}
+                
+            
+            
+            
+                com.github.ekryd.sortpom
+                sortpom-maven-plugin
+                2.4.0
+                
+                    
+                        sort-pom
+                        compile
+                        
+                            sort
+                        
+                    
+                
+                
+                    4
+                    true
+                    true
+                    ${project.encoding}
+                
+            
+            
+                com.mycila
+                license-maven-plugin
+                3.0
+                
+                    
+                        generate-sources
+                        
+                            remove
+                            format
+                        
+                    
+                
+                
+                    true
+                    
    ${main.user.dir}/tools/codestyle/HEADER
    
+                    
+                        **/src/main/java/**
+                        **/src/test/java/**
+                    
+                    
+                        **/AbstractEntry.java
+                        **/ConcurrentAutoTable.java
+                        **/NonBlockingHashMap.java
+                        **/NonBlockingHashMapLong.java
+                        **/ByteObjectHashMap.java
+                        **/Recyclers.java
+                        **/HashedWheelTimer.java
+                        **/Timeout.java
+                        **/TimerTask.java
+                        **/NonReentrantLock.java
+                        **/UnsafeUtf8Util.java
+                        **/HeapByteBufUtil.java
+                    
+                    true
+                    
+                        SLASHSTAR_STYLE
+                    
+                    
+                
+            
+        
+
+    
+    
+        
+            release
+            
+                
+                    
+                        org.apache.maven.plugins
+                        maven-javadoc-plugin
+                        2.9.1
+                        
+                            
+                                attach-javadocs
+                                
+                                    jar
+                                
+                                
+                                    -Xdoclint:none
+                                
+                            
+                        
+                    
+                    
+                        org.sonatype.plugins
+                        nexus-staging-maven-plugin
+                        1.6.7
+                        true
+                        
+                            ossrh
+                            https://oss.sonatype.org/
+                            false
+                        
+                    
+                    
+                        org.apache.maven.plugins
+                        maven-gpg-plugin
+                        1.5
+                        
+                            
+                                sign-artifacts
+                                verify
+                                
+                                    sign
+                                
+                            
+                        
+                    
+                
+            
+            
+                
+                    ossrh
+                    https://oss.sonatype.org/service/local/staging/deploy/maven2/
+                
+                
+                    ossrh
+                    https://oss.sonatype.org/content/repositories/snapshots
+                
+            
+        
+    
+    


+ * {@link HashedWheelTimer} is based on
+ * George Varghese and
+ * Tony Lauck's paper,
+ * 'Hashed
+ * and Hierarchical Timing Wheels: data structures to efficiently implement a
+ * timer facility'.  More comprehensive slides are located
+ * here.
+ *